site_graphlogo
  -   Terms of Use and Privacy
Source Code
site_graphlogo
  -   Terms of Use and Privacy
Source Code

Source Code | Data Flow Diagram GRa(F) generation

This is the main DFD graph routine behind the data flow diagrams on CruftBuster.

#!/usr/bin/python3
#Data Flow Diagram GRa(F) generation
#Copyright (C) 2020 Tributary Software

#This program is free software; you can redistribute it and/or
#modify it under the terms of the GNU General Public License
#as published by the Free Software Foundation; either version 2
#of the License, or (at your option) any later version.

#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#GNU General Public License for more details.

#You should have received a copy of the GNU General Public License
#along with this program; if not, write to the Free Software
#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#or view the license here: https://tributarysoftware.com/gpl2.txt
import re
import os
import sys
import time
import ast
import codecs
import pprint
import collections
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
diaghead='digraph {\ncharset=\"utf-8\";'
diaghead+='overlap=\"false\";\n'
diaghead+='splines=\"true\";\n'
#diaghead+='sep=\"+20\";\n'
diaghead+='sep=\"+10\";\n'
diaghead+='node [shape=record];\n'
hd=''
out='/home/knoppix/websites/site/'+sys.argv[1]+'/graph/'
rootdfd='/home/knoppix/websites/source/'+sys.argv[1]+'/'
version=''
useartdates=True
domainurl=''
graphs_ignore=''
from SPARQLWrapper import SPARQLWrapper, JSON
sparql = SPARQLWrapper("http://localhost/sparql")
sparql.addDefaultGraph("http://localhost/dfd")
sparql.setReturnFormat(JSON)
queryString = '\nSELECT ?s WHERE {?s <http://purl.obolibrary.org/obo/BFO_0000067> <http://purl.obolibrary.org/obo/OBI_0000011#BST> . }'
sparql.setQuery(queryString)
results = sparql.query().convert()
res=[]
sfrel={}
for result in results['results']['bindings']:
   res.append(result['s']['value'])
for r in res:
   sfrel[re.search('#(.+)$',r).group(1).replace('.','/')]='color="#ff3399"'
   if r.find('http://purl.obolibrary.org/obo/OBI_0000011')!=-1:
      queryString = 'SELECT ?o WHERE { <'+r+'> <http://www.w3.org/2000/01/rdf-schema#subClassOf> ?o OPTION (TRANSITIVE,T_DISTINCT,T_NO_CYCLES,T_MIN(0)) . }'
      sparql.setQuery(queryString)
      resultsr = sparql.query().convert()
      resr=[]
      for resultr in resultsr['results']['bindings']:
         resr.append(resultr['o']['value'])
      for rr in resr:
            print(rr)
            sfrel[re.search('#(.+)$',rr).group(1).replace('.','/')]='color="#ff3399"'
pprint.pprint(sfrel)
if os.path.isfile(rootdfd+'details.txt'):
   with open(rootdfd+'details.txt') as f:
      domainurl=f.readline().rstrip()
      useartdates=ast.literal_eval(f.readline().rstrip())
      graphs_ignore=f.readline().rstrip()
try:
   with open(rootdfd+'version.txt','r') as ver:
      version=ver.read()
except:
   version=''
def rendergv(level,nav):
   print(nav)
   fil=level.replace('/','.')
   if nav:
      os.system('sfdp -Tsvg '+out+fil+'.nav.gv -o '+out+fil+'.o.svg')
   else:
      os.system('sfdp -Tsvg '+out+fil+'.gv -o '+out+fil+'.o.svg')
   os.system('inkscape '+out+fil+'.o.svg -l '+out+fil+'.svg')
   fff = codecs.open(out+fil+'.svg', mode='r', encoding='utf-8')
   rawsvg=fff.read().replace('<?xml version="1.0" encoding="UTF-8" standalone="no"?>','<div id="container" style="width: 100%; height: 95vh;">')
   rawsvg=rawsvg.replace('fill="#ffffff"','fill="none"')
   rawsvg=rawsvg.replace('fill="#000000"','fill="currentColor"')
   rawsvg=rawsvg.replace('fill="#012312"','fill="currentColor"')
   rawsvg=rawsvg.replace('stroke:#012312','stroke:currentColor')
   rawsvg=rawsvg.replace('stroke:none','stroke:currentColor')
   rawsvg=rawsvg.replace('xmlns:dc="http://purl.org/dc/elements/1.1/"\n','')
   rawsvg=rawsvg.replace('xmlns:cc="http://creativecommons.org/ns#"\n','')
   rawsvg=rawsvg.replace('xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"\n','')
   rawsvg=rawsvg.replace('xmlns:svg="http://www.w3.org/2000/svg"\n','')
   rawsvg=rawsvg.replace('xmlns:xlink="http://www.w3.org/1999/xlink"\n','')
   rawsvg=rawsvg.replace('<svg\n','<svg id="Graph"\n')
   rawsvg=rawsvg.replace('stroke="#000000"','stroke="currentColor"')
   rawsvg=rawsvg.replace('font-size="14.00"\n','')
   rawsvg=rawsvg.replace('font-family="Times,serif"\n','')
   rawsvg=re.sub('id="svg.+"','',rawsvg)
   rawsvg=re.sub('height=".+"','',rawsvg)
   rawsvg=re.sub('width=".+"','',rawsvg)
   rawsvg=re.sub('<metadata.+</metadata>','',rawsvg,flags=re.DOTALL)
   fff.close
   rawsvg2=rawsvg.replace('<div id="container" style="width: 100%; height: 95vh;">','')
   fff = codecs.open(out+fil+'.e.svg', mode='w', encoding='utf-8')
   fff.write(rawsvg2)
   fff.close
   header=""
   if not nav:
      try:
         numlev=level[level.rfind('/')+1:]
         numrest=level[:level.rfind('/')]
         header=hd
         headingraw=rootdfd+numrest+'/'+numlev+'.title.txt'
         with open(headingraw,'r') as hdraw:
            headr=hdraw.read()
         heading=headr.replace('\n',' ')
      except:
         header='<a href="/">0</a>'
         with open(rootdfd+'title.txt','r') as hdraw:
            headr=hdraw.read()
         heading=headr.replace('\n',' ')
      f = codecs.open(out+fil+'.txt', mode='w', encoding='utf-8')
      tp='<div class="rhead"><div class="parent"><div class="level">'+header+'</div><div class="heading">'+heading+'</div><div class="version">v.'+version+'</div></div></div>'
      tp+='<div class="rhead-push"><div class="parent"><div class="level">'+header+'</div><div class="heading">'+heading+'</div><div class="version">v.'+version+'</div></div></div>'
      f.write(tp+rawsvg)
      f.close
   else:
      head='''<!doctype html>
<html lang='en'>
<head>
<meta HTTP-EQUIV='Content-Type' CONTENT='text/html; charset=utf-8'>
<link rel="stylesheet" type="text/css" href="/css/style.css" />
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="/svg-pan-zoom.min.js"></script>
</head>
<body>
'''
      foot='''
<script>
      window.onload = function() {
        window.zoomGraph = svgPanZoom('#Graph', {
          zoomEnabled: true,
          controlIconsEnabled: false,
          fit: true,
          center: true,
        });
      zoomGraph.zoomBy(.5);
      };
    </script>
</body></html>
'''
      f = codecs.open(out+fil+'.nav.html', mode='w', encoding='utf-8')
      f.write(head+rawsvg+foot)
def createlevels(level):
   print('creating level: '+level)
   labels={}
   labelsn={}
   global hd
   graph=[]
   for root,dirs,files in os.walk(rootdfd+level+'/', followlinks=False):
      for fname in files:
         if not fname.startswith('.') and fname[-13:]!='predlabel.txt' and fname[-11:]!='details.txt' and fname.find('^^')==-1 and fname.find('_component_destination')==-1 and fname.find('_flow_destination')==-1:
            p=os.path.join(root,fname)
            bits=re.compile(re.escape(rootdfd+level+'/')+r'(\d+)/(has_specified_input|has_specified_output|has_specified_inp_and_out)/(.+)')
            try:
               m = bits.match(p)
               subject=m.group(1)
               print('subject: '+subject)
               predicate=m.group(2)
               objctraw=m.group(3)
               ttl=objctraw.find('title.txt')
               if ttl==-1:
                  objct=objctraw[:-4]
                  objctp=objctraw
               else:
                  objct=objctraw[:ttl-1]
                  objctp=objctraw
               if os.path.isfile(rootdfd+level+'/'+subject+'/'+predicate+'/'+objct+'.predlabel.txt'):
                  with open(rootdfd+level+'/'+subject+'/'+predicate+'/'+objct+'.predlabel.txt','r') as linelbl:
                        lbllinel=linelbl.read()
               else:
                  lbllinel=''
               if os.path.isfile(rootdfd+level+'/'+subject+'/'+predicate+'/'+objct+'.details.txt'):
                  with open(rootdfd+level+'/'+subject+'/'+predicate+'/'+objct+'.details.txt','r') as linelbl:
                     lblline=linelbl.read().replace('\n',' ')
                  as_follows=': '
               else:
                  lblline=''
                  as_follows=''
               if lbllinel=='*':
                  fontsz='32'
               else:  
                  fontsz='16'
               if predicate=='has_specified_inp_and_out':
                  end='[dir="both" label="'+lbllinel+'" labeltooltip="'+subject+' <-> '+objct+as_follows+lblline+'" fontsize='+fontsz+'];'
               if predicate=='has_specified_output':
                  end='[label="'+lbllinel+'" labeltooltip="'+subject+' -> '+objct+as_follows+lblline+'" fontsize='+fontsz+'];'
               if predicate=='has_specified_input':
                  end='[dir="back" label="'+lbllinel+'" labeltooltip="'+subject+'<-'+objct+as_follows+lblline+'" fontsize='+fontsz+'];'
               graph.append('"'+subject+'" -> "'+objct+'"'+end)
               print('"'+subject+'" -> "'+objct+'"'+end)
               mo=re.search('\d|\.',objct)
               lbl=''
               with open(rootdfd+level+'/'+objctp,'r') as label:
                  for line in label:
                     lbl+=line.rstrip()+r'\n'
               try:
                  with open(rootdfd+level+'/'+objct+'.details.txt','r') as linelbl:
                     lblline=linelbl.read().replace('\n',' ')
                     if len(lblline)>0:
                        lbllinel='*'
                     else:
                        lbllinel=''
               except:
                  lblline=''
                  lbllinel=''
               end=('tooltip="'+lblline+'"];')
               if objct.find('D')!=-1 and mo:
                  if level=='0':
                     labels[objct]='"'+objct+'" [label="<f0>'+lbllinel+' '+objct+'|<f1> '+lbl+'" '+end
                     labelsn[objct]='"'+objct+'" [href="/'+objct+'/" fontcolor="#012312" label="<f0>'+lbllinel+' '+objct+'|<f1> '+lbl+'" tooltip="-" ];'
                  else:
                     labels[objct]='"'+objct+'" [label="<f0>'+lbllinel+' '+objct+'|<f1> '+lbl+'" '+end
                     labelsn[objct]='"'+objct+'" [href="/'+level[2:]+'/'+objct+'/" fontcolor="#012312" label="<f0>'+lbllinel+' '+objct+'|<f1> '+lbl+'" tooltip="-" ];'
               elif mo:
                  if os.path.isfile(out+level.replace('/','.')+'.'+objct+'.gv'):
                     if level=='0':
                        labels[objct]='"'+objct+'" [href="/'+objct+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+objct+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord '+end
                        labelsn[objct]='"'+objct+'" [href="/'+objct+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+objct+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord  tooltip="-" ];'
                     else:
                        labels[objct]='"'+objct+'" [href="/'+level[2:]+'/'+objct+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+objct+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord '+end
                        labelsn[objct]='"'+objct+'" [href="/'+level[2:]+'/'+objct+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+objct+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord  tooltip="-" ];'
                  else:
                     labels[objct]='"'+objct+'" [fontcolor="#012312" label="{<f0> '+lbllinel+' '+objct+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord '+end
                     if level=='0':
                        labelsn[objct]='"'+objct+'" [href="/'+objct+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+objct+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord  tooltip="-" ];'
                     else:
                        labelsn[objct]='"'+objct+'" [href="/'+level[2:]+'/'+objct+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+objct+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord  tooltip="-" ];'
               if not(objct.find('D')!=-1 and mo) and not mo:
                  if level=='0':
                     labels[objct]='"'+objct+'" [label="'+lbllinel+' '+lbl+'" shape=box '+end
                     labelsn[objct]='"'+objct+'" [href="/'+objct+'/" fontcolor="#012312" label="'+lbllinel+' '+lbl+'" shape=box  tooltip="-" ];'
                  else:
                     labels[objct]='"'+objct+'" [label="'+lbllinel+' '+lbl+'" shape=box '+end
                     labelsn[objct]='"'+objct+'" [href="/'+level[2:]+'/'+objct+'/" fontcolor="#012312" label="'+lbllinel+' '+lbl+'" shape=box  tooltip="-" ];'
               lbl=''
               with open(rootdfd+level+'/'+subject+'.title.txt','r') as label:
                  for line in label:
                     lbl+=line.rstrip()+r'\n'
               try:
                  with open(rootdfd+level+'/'+subject+'.details.txt','r') as linelbl:
                     lblline=linelbl.read().replace('\n',' ')
                     if len(lblline)>0:
                        lbllinel='*'
                     else:
                        lbllinel=''
               except:
                  lblline=''
                  lbllinel=''
               end=('tooltip="'+lblline+'"];')
               if os.path.isfile(out+level.replace('/','.')+'.'+subject+'.gv'):
                  print('lvel: '+level+'/'+subject)
                  try:
                     fc=sfrel[level+'/'+subject]
                  except:
                     fc=''
                  if level=='0':
                     labels[subject]='"'+subject+'" [href="/'+subject+'/" '+fc+' fontcolor="#012312" label="{<f0> '+lbllinel+' '+subject+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord '+end
                     labelsn[subject]='"'+subject+'" [href="/'+subject+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+subject+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord  tooltip="-" ];'
                  else:
                     labels[subject]='"'+subject+'" [href="/'+level[2:]+'/'+subject+'/" '+fc+' fontcolor="#012312" label="{<f0> '+lbllinel+' '+subject+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord '+end
                     labelsn[subject]='"'+subject+'" [href="/'+level[2:]+'/'+subject+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+subject+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord  tooltip="-" ];'
               else:
                  labels[subject]='"'+subject+'" [label="{<f0> '+lbllinel+' '+subject+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord '+end
                  labelsn[subject]='"'+subject+'" [href="/'+level[2:]+'/'+subject+'/" fontcolor="#012312" label="{<f0> '+lbllinel+' '+subject+'|<f1> '+lbl+'\n\n\n}" shape=Mrecord  tooltip="-" ];'
            except:
               six=9
   if level=='0':
      hd='/'
   else:
      head=level[2:].split('/')
      cumulative=''
      hcumulative='<a href="/">0</a>'
      for bit in head:
         cumulative+=bit+'/'
         hcumulative+='.<a href="/'+cumulative+'">'+bit+'</a>'
      hd=hcumulative
   alll=''



   for line in labels:
      alll+=labels[line]+'\n'
   for line in graph:
      alll+=line+'\n'
   with open(out+level.replace('/','.')+'.gv', 'w+') as fa:
      fa.write(diaghead+alll+'}'+'\n')
   alll=''
   rendergv(level,False)
   for line in labelsn:
      alll+=labelsn[line]+'\n'
   for line in graph:
      alll+=line+'\n'
   with open(out+level.replace('/','.')+'.nav.gv', 'w+') as fa:
      fa.write(diaghead+alll+'}'+'\n')
   rendergv(level,True)
def update(evt):
   print(evt)
   proc=''
   if evt.find(
   graphs_ignore)==-1 and evt.find(
   '.html')==-1 and evt.find(
   '^^')==-1:
      hass=evt.find('/has_specified')
      if hass !=-1:
         proc=evt[:hass].rfind('/')
      else:
         proc=evt.rfind('/')
      global version
      if evt!=rootdfd+'version.txt' and evt!=rootdfd+'title.txt':
         createlevels(evt[len(rootdfd):proc])
         try:
            with open(rootdfd+'version.txt','r') as ver:
               v=int(version)
               version=str(v+1)
               fa = codecs.open(rootdfd+'version.txt', mode='w', encoding='utf-8')
               fa.write(version)
               fa.close
         except:
            version=''
      elif evt==rootdfd+'title.txt':
         l={}
         l.clear()
         for root,dirs,files in os.walk(rootdfd, followlinks=False):
            for fname in files:
               p=os.path.join(root,fname)
               bits=re.compile(re.escape(rootdfd)+r'(.+)/\d+/(has_specified_input|has_specified_output|has_specified_inp_and_out)/.+')
               try:
                  m = bits.match(p)
                  subject=m.group(1)
                  l[subject]=''
               except:
                  six=9
         try:
            for lvl in l:
               createlevels(lvl)
         except:
            six=9
class changes(FileSystemEventHandler):
   def on_modified(self, event):
      update(event.src_path)
   def on_created(self, event):
      update(event.src_path)
if __name__ == "__main__":
   event_handler = changes()
   observer = Observer()
   observer.schedule(event_handler, rootdfd, recursive=True)
   observer.start()
   try:
      while True:
         time.sleep(1)
   except KeyboardInterrupt:
      observer.stop()
      observer.join()

raw source