Translations of this page?:

This is an old revision of the document!

grapher.php 
  1. #!/usr/bin/php
  2. <?php
  3. if(!defined('DOKU_INC')) define('DOKU_INC', realpath(dirname(__FILE__) . '/../') . '/');
  4. require_once(DOKU_INC . 'inc/init.php');
  5.  
  6. class Grapher extends DokuCLI {
  7.  
  8.     /**
  9.      * Register options and arguments on the given $options object
  10.      *
  11.      * @param DokuCLI_Options $options
  12.      * @return void
  13.      */
  14.     protected function setup(DokuCLI_Options $options) {
  15.         $options->setHelp('Creates a graph representation of pages and media files and how they are interlinked.');
  16.         $options->registerOption(
  17.             'depth',
  18.             'Recursion depth, eg. how deep to look into the given namespaces. Use 0 for all. Default: 1',
  19.             'd', 'depth');
  20.         $options->registerOption(
  21.             'media',
  22.              "How to handle media files. 'ns' includes only media that is located in the given namespaces, ".
  23.              "'all' includes all media files and 'none' ignores the media files completely. ".
  24.              "Default: ns",
  25.             'm', 'ns|all|none');
  26.         $options->registerOption(
  27.             'format',
  28.             "The wanted output format. 'dot' is a very simple format which can be used to visualize the resulting ".
  29.             "graph with graphviz. The 'gexf' format is a more complex XML-based format which contains more info ".
  30.             "about the found nodes and can be loaded in Gephi. Default: dot",
  31.             'f', 'dot|gexf|json');
  32.         $options->registerOption(
  33.             'output',
  34.             "Where to store the output eg. a filename. If not given the output is written to STDOUT.",
  35.             'o', 'file');
  36.         $options->registerArgument(
  37.             'namespaces',
  38.             "Give all wiki namespaces you want to have graphed. If no namespace is given, the root ".
  39.             "namespace is assumed.",
  40.             false
  41.         );
  42.     }
  43.  
  44.     /**
  45.      * Your main program
  46.      *
  47.      * Arguments and options have been parsed when this is run
  48.      *
  49.      * @param DokuCLI_Options $options
  50.      * @return void
  51.      */
  52.     protected function main(DokuCLI_Options $options) {
  53.         $depth = $options->getOpt('depth', 1);
  54.         $media = $options->getOpt('media', 'ns');
  55.         if(!in_array($media, array('ns', 'all', 'none'))) {
  56.             $this->fatal('Bad media option: ' . $media);
  57.         }
  58.         $format = $options->getOpt('format', 'dot');
  59.         if(!in_array($format, array('dot', 'gexf','json'))) {
  60.             $this->fatal('Bad format option: ' . $format);
  61.         }
  62.         $output = $options->getOpt('output', '-');
  63.         if($output == '-') $output = 'php://stdout';
  64.  
  65.         $namespaces = array_map('cleanID', $options->args);
  66.         if(!count($namespaces)) $namespaces = array(''); //import from top
  67.  
  68.         $fh = @fopen($output, 'w');
  69.         if(!$fh) $this->fatal("Failed to open $output");
  70.  
  71.         $data = $this->gather_data($namespaces, $depth, $media);
  72.         if($format == 'dot') {
  73.             $this->create_dot($data, $fh);
  74.         } elseif($format == 'gexf') {
  75.             $this->create_gexf($data, $fh);
  76.         } elseif($format == 'json') {
  77.             $this->create_json($data, $fh);
  78. 		}
  79.         fclose($fh);
  80.     }
  81.  
  82.     /**
  83.      * Find all the node and edge data for the given namespaces
  84.      * @param $namespaces
  85.      * @param int $depth
  86.      * @param string $incmedia
  87.      * @return array
  88.      */
  89.     protected function gather_data($namespaces, $depth = 0, $incmedia = 'ns') {
  90.         global $conf;
  91.         /** @var helper_plugin_translation $transplugin */
  92.         $transplugin = plugin_load('helper', 'translation');
  93.  
  94.         $pages = array();
  95.         $media = array();
  96.         foreach($namespaces as $ns) {
  97.             // find media
  98.             if($incmedia == 'ns') {
  99.                 $data = array();
  100.                 search(
  101.                     $data,
  102.                     $conf['mediadir'],
  103.                     'search_universal',
  104.                     array(
  105.                         'depth' => $depth,
  106.                         'listfiles' => true,
  107.                         'listdirs' => false,
  108.                         'pagesonly' => false,
  109.                         'skipacl' => true,
  110.                         'keeptxt' => true,
  111.                         'meta' => true,
  112.                     ),
  113.                     str_replace(':', '/', $ns)
  114.                 );
  115.  
  116.                 // go through all those media files
  117.                 while($item = array_shift($data)) {
  118.                     $media[$item['id']] = array(
  119.                         'title' => noNS($item['id']),
  120.                         'size' => $item['size'],
  121.                         'ns' => getNS($item['id']),
  122.                         'time' => $item['mtime'],
  123.                     );
  124.                 }
  125.             }
  126.  
  127.             // find pages
  128.             $data = array();
  129.             search(
  130.                 $data,
  131.                 $conf['datadir'],
  132.                 'search_universal',
  133.                 array(
  134.                     'depth' => $depth,
  135.                     'listfiles' => true,
  136.                     'listdirs' => false,
  137.                     'pagesonly' => true,
  138.                     'skipacl' => true,
  139.                     'firsthead' => true,
  140.                     'meta' => true,
  141.                 ),
  142.                 str_replace(':', '/', $ns)
  143.             );
  144.  
  145.             // ns start page
  146.             if($ns && page_exists($ns)) {
  147.                 $data[] = array(
  148.                     'id' => $ns,
  149.                     'ns' => getNS($ns),
  150.                     'title' => p_get_first_heading($ns, false),
  151.                     'size' => filesize(wikiFN($ns)),
  152.                     'mtime' => filemtime(wikiFN($ns)),
  153.                     'perm' => 16,
  154.                     'type' => 'f',
  155.                     'level' => 0,
  156.                     'open' => 1,
  157.                 );
  158.             }
  159.  
  160.             // go through all those pages
  161.             while($item = array_shift($data)) {
  162.                 $time = (int) p_get_metadata($item['id'], 'date created', false);
  163.                 if(!$time) $time = $item['mtime'];
  164.                 $lang = ($transplugin) ? $transplugin->getLangPart($item['id']) : '';
  165.  
  166.                 if($lang) $item['ns'] = preg_replace('/^' . $lang . '(:|$)/', '', $item['ns']);
  167.  
  168.                 $pages[$item['id']] = array(
  169.                     'title' => $item['title'],
  170.                     'ns' => $item['ns'],
  171.                     'size' => $item['size'],
  172.                     'time' => $time,
  173.                     'links' => array(),
  174.                     'linklabel' => array(),
  175.                     'media' => array(),
  176.                     'lang' => $lang
  177.                 );
  178.             }
  179.         }
  180.  
  181.         // now get links and media
  182.         foreach($pages as $pid => $item) {
  183.             // get instructions
  184.             $ins = p_cached_instructions(wikiFN($pid), false, $pid);
  185.             // find links and media usage
  186.             foreach($ins as $i) {
  187.                 $mid = null;
  188.  
  189.                 if($i[0] == 'internallink') {
  190.                     $id = $i[1][0];
  191.                     $label = $i[1][1];
  192.                     $exists = true;
  193.                     resolve_pageid($item['ns'], $id, $exists);
  194.                     list($id) = explode('#', $id, 2);
  195.                     if($id == $pid) continue; // skip self references
  196.                     if($exists && isset($pages[$id])) {
  197.                         $pages[$pid]['links'][] = $id;
  198.                         $pages[$pid]['linklabel'][] = $label;
  199.                     }
  200.                     if(is_array($i[1][1]) && $i[1][1]['type'] == 'internalmedia') {
  201.                         $mid = $i[1][1]['src']; // image link
  202.                     } else {
  203.                         continue; // we're done here
  204.                     }
  205.                 }
  206.  
  207.                 if($i[0] == 'internalmedia') {
  208.                     $mid = $i[1][0];
  209.                 }
  210.  
  211.                 if(is_null($mid)) continue;
  212.                 if($incmedia == 'none') continue; // no media wanted
  213.  
  214.                 $exists = true;
  215.                 resolve_mediaid($item['ns'], $mid, $exists);
  216.                 list($mid) = explode('#', $mid, 2);
  217.                 $mid = cleanID($mid);
  218.  
  219.                 if($exists) {
  220.                     if($incmedia == 'all') {
  221.                         if(!isset($media[$mid])) { //add node
  222.                             $media[$mid] = array(
  223.                                 'size' => filesize(mediaFN($mid)),
  224.                                 'time' => filemtime(mediaFN($mid)),
  225.                                 'ns' => getNS($mid),
  226.                                 'title' => noNS($mid),
  227.                             );
  228.                         }
  229.                         $pages[$pid]['media'][] = $mid;
  230.                     } elseif(isset($media[$mid])) {
  231.                         $pages[$pid]['media'][] = $mid;
  232.                     }
  233.                 }
  234.             }
  235.  
  236.             // clean up duplicates
  237.             $pages[$pid]['links'] = array_unique($pages[$pid]['links']);
  238.             $pages[$pid]['media'] = array_unique($pages[$pid]['media']);
  239.         }
  240.  
  241.         return array('pages' => $pages, 'media' => $media);
  242.  
  243.  
  244.     }
  245.  
  246.     /**
  247.      * Create a Graphviz dot representation
  248.      *
  249.      * @param array $data
  250.      * @param resource $fh
  251.      */
  252.     protected function create_dot(&$data, $fh) {
  253.         $pages =& $data['pages'];
  254.         $media =& $data['media'];
  255.  
  256.         fwrite($fh, "digraph G {\n");
  257.         fwrite($fh, "labelloc=\"b\";\nlabel=\"Actors of a Persecution - https:\/\/challengepower.info - #FreeAssange\";");   
  258.         fwrite($fh, "bgcolor=\"white\";
  259. node [shape = \"circle\", fontsize=16,	color=black, bgcolor=aquamarine];
  260. edge [arrowsize=2, color=black];
  261. root=\"page-the_actors:assange\";
  262.  
  263. subgraph corporations {
  264.     	\"page-the_actors:corporations:darktrace\" , 
  265. 	\"page-the_actors:corporations:thales\" , 
  266. 	\"page-the_actors:corporations:sc_strategy_limited\", 
  267. 	\"page-the_actors:corporations:startfor\",
  268. 	\"page-the_actors:corporations:uc_global_s.l\"
  269. }
  270.  
  271. subgraph leaks {
  272. 	\"page-the_actors:leaks:global_intelligences_files\" , 
  273. 	\"page-the_actors:leaks:spy_files\", 
  274. 	\"page-the_actors:leaks:vault7\"
  275. }
  276.  
  277. subgraph countries {
  278. 	\"page-the_actors:ukgov:start\",
  279. 	\"page-the_actors:usgov:start\",
  280. 	\"page-the_actors:sweden:start\",
  281. 	\"page-the_actors:ecuador:start\"
  282.  
  283. }
  284. \n");
  285.  
  286.         // create all nodes first
  287.         foreach($pages as $id => $page) {
  288.             fwrite($fh, "  \"page-$id\" [ 
  289.             shape=note,
  290.             label=\"{$page['title']}\",\n");
  291.             // look for links
  292.             if (!empty( $page['links'][0]))
  293. 					fwrite($fh,"URL=\"/". str_replace(':','/',$page['links'][0]) ."\"");
  294. 			else 	fwrite($fh,"URL=\"#\"");
  295. 			// look for media
  296.             if (!empty( $page['media'][0]))
  297.             {
  298. 				$path = explode (":", $page['media'][0]);
  299. 				$file = count($path );
  300. 				$folder="";
  301. 				for($i=0 ; $i < $file-1 ; $i++)
  302. 				{
  303. 					$folder = $folder . $path[$i] . "/";
  304. 				}
  305. 				fwrite($fh,"\n // " . $page['media'][0] . " ### " . $file ." ### ". $path[0] . " ### ". $path[1]);
  306. 				fwrite($fh,",\n           labelloc=\"b\",\n          shapefile=\"".DOKU_INC. "data/media/" . $folder . "300x300-" . $path[$file-1] . "\"");
  307. 			}
  308. 			fwrite($fh,";\n ];\n");
  309.         }
  310.                  //  var_dump($pages);
  311. /*
  312.        foreach($media as $id => $item) {
  313.             fwrite($fh, "    \"media-$id\" [shape=box, label=\"$id\", color=sandybrown, fontname=Helvetica];\n");
  314.         }
  315.   */      
  316.         // now create all the links
  317.         foreach($pages as $id => $page) {
  318. 			for($n=0;$n<count($page['links']);$n++)
  319. 				{
  320. 				fwrite($fh, "    \"page-". $id ."\" -> \"page-". $page['links'][$n] ."\" [color=navy;label=\"". $page['linklabel'][$n] ."\"];\n");
  321. 			}
  322.         }
  323.         fwrite($fh, "}\n");
  324.     }
  325.  
  326.     /**
  327.      * Create a GEXF representation
  328.      *
  329.      * @param array $data
  330.      * @param resource $fh
  331.      */
  332.     protected function create_gexf(&$data, $fh) {
  333.         $pages =& $data['pages'];
  334.         $media =& $data['media'];
  335.  
  336.         fwrite($fh, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
  337.         fwrite(
  338.             $fh, "<gexf xmlns=\"http://www.gexf.net/1.1draft\" version=\"1.1\"
  339.                    xmlns:viz=\"http://www.gexf.net/1.1draft/viz\">\n"
  340.         );
  341.         fwrite($fh, "    <meta lastmodifieddate=\"" . date('Y-m-d H:i:s') . "\">\n");
  342.         fwrite($fh, "        <creator>DokuWiki</creator>\n");
  343.         fwrite($fh, "    </meta>\n");
  344.         fwrite($fh, "    <graph mode=\"dynamic\" defaultedgetype=\"directed\">\n");
  345.  
  346.         // define attributes
  347.         fwrite($fh, "        <attributes class=\"node\">\n");
  348.         fwrite($fh, "            <attribute id=\"title\" title=\"Title\" type=\"string\" />\n");
  349.         fwrite($fh, "            <attribute id=\"lang\" title=\"Language\" type=\"string\" />\n");
  350.         fwrite($fh, "            <attribute id=\"ns\" title=\"Namespace\" type=\"string\" />\n");
  351.         fwrite($fh, "            <attribute id=\"type\" title=\"Type\" type=\"liststring\">\n");
  352.         fwrite($fh, "                <default>page|media</default>\n");
  353.         fwrite($fh, "            </attribute>\n");
  354.         fwrite($fh, "            <attribute id=\"time\" title=\"Created\" type=\"long\" />\n");
  355.         fwrite($fh, "            <attribute id=\"size\" title=\"File Size\" type=\"long\" />\n");
  356.         fwrite($fh, "        </attributes>\n");
  357.  
  358.         // create all nodes first
  359.         fwrite($fh, "        <nodes>\n");
  360.         foreach($pages as $id => $item) {
  361.             $title = htmlspecialchars($item['title']);
  362.             $lang = htmlspecialchars($item['lang']);
  363.             fwrite($fh, "            <node id=\"page-$id\" label=\"$id\" start=\"{$item['time']}\">\n");
  364.             fwrite($fh, "               <attvalues>\n");
  365.             fwrite($fh, "                   <attvalue for=\"type\" value=\"page\" />\n");
  366.             fwrite($fh, "                   <attvalue for=\"title\" value=\"$title\" />\n");
  367.             fwrite($fh, "                   <attvalue for=\"lang\" value=\"$lang\" />\n");
  368.             fwrite($fh, "                   <attvalue for=\"ns\" value=\"{$item['ns']}\" />\n");
  369.             fwrite($fh, "                   <attvalue for=\"time\" value=\"{$item['time']}\" />\n");
  370.             fwrite($fh, "                   <attvalue for=\"size\" value=\"{$item['size']}\" />\n");
  371.             fwrite($fh, "               </attvalues>\n");
  372.             fwrite($fh, "               <viz:shape value=\"square\" />\n");
  373.             fwrite($fh, "               <viz:color r=\"173\" g=\"216\" b=\"230\" />\n");
  374.             fwrite($fh, "            </node>\n");
  375.         }
  376.         foreach($media as $id => $item) {
  377.             $title = htmlspecialchars($item['title']);
  378.             $lang = htmlspecialchars($item['lang']);
  379.             fwrite($fh, "            <node id=\"media-$id\" label=\"$id\" start=\"{$item['time']}\">\n");
  380.             fwrite($fh, "               <attvalues>\n");
  381.             fwrite($fh, "                   <attvalue for=\"type\" value=\"media\" />\n");
  382.             fwrite($fh, "                   <attvalue for=\"title\" value=\"$title\" />\n");
  383.             fwrite($fh, "                   <attvalue for=\"lang\" value=\"$lang\" />\n");
  384.             fwrite($fh, "                   <attvalue for=\"ns\" value=\"{$item['ns']}\" />\n");
  385.             fwrite($fh, "                   <attvalue for=\"time\" value=\"{$item['time']}\" />\n");
  386.             fwrite($fh, "                   <attvalue for=\"size\" value=\"{$item['size']}\" />\n");
  387.             fwrite($fh, "               </attvalues>\n");
  388.             fwrite($fh, "               <viz:shape value=\"disc\" />\n");
  389.             fwrite($fh, "               <viz:color r=\"244\" g=\"164\" b=\"96\" />\n");
  390.             fwrite($fh, "            </node>\n");
  391.         }
  392.         fwrite($fh, "        </nodes>\n");
  393.  
  394.         // now create all the edges
  395.         fwrite($fh, "        <edges>\n");
  396.         $cnt = 0;
  397.         foreach($pages as $id => $page) {
  398.             foreach($page['links'] as $link) {
  399.                 $cnt++;
  400.                 fwrite($fh, "            <edge id=\"$cnt\" source=\"page-$id\" target=\"page-$link\" />\n");
  401.             }
  402.             foreach($page['media'] as $link) {
  403.                 $cnt++;
  404.                 fwrite($fh, "            <edge id=\"$cnt\" source=\"page-$id\" target=\"media-$link\" />\n");
  405.             }
  406.         }
  407.         fwrite($fh, "        </edges>\n");
  408.  
  409.         fwrite($fh, "    </graph>\n");
  410.         fwrite($fh, "</gexf>\n");
  411.  
  412.    }
  413.  
  414.     /* Create a JSON representation
  415.      */
  416.     protected function create_json(&$data, $fh) {
  417.  
  418.         $pages =& $data['pages'];
  419.         $media =& $data['media'];
  420. 		$root_name = array_values($pages)[0]["ns"];  
  421.  
  422.         $output = '{
  423. 			  "root": "'. $root_name .'",
  424. 			  "nodes": ['."\n" ;
  425. 		$i=0;
  426.         // create all nodes first
  427.         foreach($pages as $id => $item){
  428. 			$i++;
  429.             $title = htmlspecialchars($item['title']);			
  430.             $lang  = htmlspecialchars($item['lang']);
  431.             $output .= "{\n";
  432.             $output .= '"id" : "'. $id .'",'."\n";
  433.             $output .= '"name" : "'. $title .'",'."\n";
  434.             $output .= '"value" : "'. $item['size'] .'",'."\n";
  435.             $output .= '"type" : "'. substr($item['ns'],4) .'"'."\n";
  436.             if (!empty( $item['media'][0]))
  437.             {
  438. 				$path = explode (":", $item['media'][0]);
  439. 				$file = count($path );
  440. 				$folder="";
  441. 				for($n=0 ; $n < $file-1 ; $n++)
  442. 				{
  443. 					$folder = $folder . $path[$n] . "/";
  444. 				}
  445. 				$output .= ',"img" : "'. DOKU_INC. "data/media/" . $folder . "300x300-" . $path[$file-1] .'"'."\n";
  446. 			}
  447.            // $output .= '"has_conflict" : "'.$id.'"'."\n";
  448.            // $output .= '"description" : "'.$id.'"'."\n";
  449.             $output .= "}\n";
  450.             if ($i < count($pages)-1) $output .= ',';
  451.  
  452.         }
  453.  
  454.         // now create all the edges
  455.          $output .= '],
  456. 			"links": ['."\n";
  457.  
  458.         $cnt = 0;
  459.         foreach($pages as $id => $page){
  460.  
  461.             foreach($page['links'] as $link){
  462. 				 if ($cnt > 0) $output .= ',';
  463.  
  464. 				 $output .= '{
  465. 				  "source": "'. $id . '",
  466. 				  "target": "'. $link . '",
  467. 				  "value": "'. $page["linklabel"][$cnt] .'"
  468. 				}';
  469.                 $cnt++;
  470.             }
  471.  
  472.         }
  473.  
  474.  
  475.         $output .= " ] }";
  476.  
  477.         fwrite($fh,  $output);
  478.     }
  479.  
  480. }
  481.  
  482. $grapher = new Grapher();
  483. $grapher->run();
You could leave a comment if you were logged in.
  • dev/grapher-php.1604245495.txt.gz
  • Last modified: 2020/11/01 15:44
  • by bab