Start on normal top-down minimal parser

Author: Dmitry Olshansky

source/ast.d

@@ -1,31 +1,63 @@
 //Written in the D programming language
 // libdparse:
-import std.d.lexer;
+import dparse.lexer;
 // std
 import std.format;
-@safe:
-
-/// Symbol DAG element
-class Symbol {
-    string      file;
-    string      name;
-    Token[]     tokens;
-    bool        gcUsed;
+
+/// Element of a DAG that represent lexical nesting.
+class Ast {
+private:
+    string          file;
+    string          name;
+    const(Token)[]  tokens;
+    bool            gcUsed;
     //
-    Symbol      parent;
-    Symbol[]    children;
+    Ast             parent;
+    Ast[]           children;
+
+    void dumpEdges(scope void delegate (const(char)[]) sink){
+        foreach(c; children){
+            formattedWrite(sink, `"%s" -> "%s";\n`, name, c.name);
+            c.dumpEdges(sink);
+        }
+    }
+
+public:
+    this(string file, string name, const(Token)[] range, bool gc){
+        this.file = file;
+        this.name = name;
+        this.tokens = range;
+        this.gcUsed = gc;
+    }
 
     @property size_t firstLine(){
-        return tokens[0].line
+        return tokens[0].line;
     }
 
     @property size_t endLine(){
-        return tokens[0].line
+        return tokens[0].line;
+    }
+
+    /// Analyze GC usage of any of the nested symbols
+    bool analyzeGC(){
+        foreach(c; children){
+            if(c.analyzeGC){
+                gcUsed = true;
+                break;
+            }
+        }
+        return false;
     }
 
     /// Dumps Graphviz format of this DAG
     void dump(scope void delegate (const(char)[]) sink){
-
+        formattedWrite(sink, "graph {\n");
+        dumpEdges(sink);
+        formattedWrite(sink, "}\n");
     }
 }
 
+/// A cross-reference graph element
+class Symbol{
+
+}

source/gchunt.d

@@ -8,12 +8,12 @@
 module gchunt;
 
 import std.algorithm, std.conv, std.stdio, std.string, std.exception,
-    std.regex, std.range, std.process;
+    std.regex, std.range, std.process, std.getopt;
 
 static import std.file;
 
 // libdparse:
-import std.d.lexer;
+import dparse.lexer;
 // our pattern matching on D tokens
 import revdpattern;
 
@@ -192,9 +192,9 @@ unittest{
     assert(matchPattern("aaaafdsfbbbasd", "aaaa*b*"));
 }
 
-BlacklistEntry[] blacklist;
+BlacklistEntry[] blacklist; //
 
-Comment[] talk; // comments
+// comments
 
 struct Reason{
     string reason;
@@ -212,7 +212,8 @@ struct Artifact{
 Artifact[string] artifacts;
 
 // load comments from wiki dump
-void loadComments(string path){
+Comment[]  loadComments(string path){
+    Comment[] talk; 
     auto lines = File(path).byLine.map!(x => x.idup).array;
     string[] record;
     foreach(i, line; lines){
@@ -232,15 +233,18 @@ void loadComments(string path){
         if(m)
             record ~= m[1];
     }
+    return talk;
 }
 
 version(unittest) void main(){}
-else void main(){
-    string fmt = "mediawiki";
-    string gitHost = `https://github.com`;
-    string gitHash = gitHEAD();
-    string gitRepo = gitRemotePath();
+else void main(string[] args){
+    bool graphMode = false;
+    
     auto re = regex(`(.*[\\/]\w+)\.d\((\d+)\):\s*vgc:\s*(.*)`);
+    auto opts = getopt(args, "graph", &graphMode);
+    if(opts.helpWanted){
+        defaultGetoptPrinter("gchunt - pinpoint GC usage in D apps", opts.options);
+    }
     Result[] results;
     foreach(line; stdin.byLine){
         auto m = line.idup.matchFirst(re);
@@ -251,19 +255,15 @@ else void main(){
     sort!((a,b) => a.file < b.file || (a.file == b.file && a.line < b.line))
         (results);
 
-    results = uniq(results).array;
+    results = uniq(results).array; // deduplicate 
+    // Tokenize modules in question
     auto interned = StringCache(4096);
     foreach(mod;results.map!(x => x.file).uniq){
         auto config = LexerConfig(mod~".d", StringBehavior.compiler);
         auto data = cast(ubyte[])std.file.read(mod ~ ".d");
         tokenStreams[mod] = getTokensForParser(data, config, &interned).dup;
         //TODO: generate new "vgc" records for each .(i)dup
     }
-    try{
-        loadComments("talk.gchunt");
-        stderr.writefln("talk.gchunt loaded: %d comments.", talk.length);
-    }
-    catch(Exception){} // was that FileException?
     try{
         auto f = File("blacklist.gchunt");
         stderr.writeln("Found blacklist.gchunt ...");
@@ -276,67 +276,84 @@ else void main(){
         stderr.writefln("blacklist.gchunt loaded: %d patterns.", blacklist.length);
     }
     catch(Exception){}
-    foreach(r; results){
-        //writeln(r.file, ",", r.line, ",", r.reason);
-        auto mod = r.file.replace("/", ".");
-        auto artifact = findArtifact(r);
-        auto path = mod~":"~artifact;
-        if(path !in artifacts){
-            artifacts[path] = Artifact(artifact, mod);
+    if(graphMode){
+
+    }
+    else {
+        string fmt = "mediawiki";
+        string gitHost = `https://github.com`;
+        string gitHash = gitHEAD();
+        string gitRepo = gitRemotePath();
+        Comment[] talk;
+        try{
+            talk = loadComments("talk.gchunt");
+            stderr.writefln("talk.gchunt loaded: %d comments.", talk.length);
         }
-        auto idx = artifacts[path].reasons.countUntil!(x => x.reason == r.reason);
-        if(idx < 0) {
-            artifacts[path].reasons ~= Reason(r.reason, [to!int(r.line)]);
+        catch(Exception){} // was that FileException?   
+        foreach(r; results){
+            //writeln(r.file, ",", r.line, ",", r.reason);
+            auto mod = r.file.replace("/", ".");
+            auto artifact = findArtifact(r);
+            auto path = mod~":"~artifact;
+            if(path !in artifacts){
+                artifacts[path] = Artifact(artifact, mod);
+            }
+            auto idx = artifacts[path].reasons.countUntil!(x => x.reason == r.reason);
+            if(idx < 0) {
+                artifacts[path].reasons ~= Reason(r.reason, [to!int(r.line)]);
+            }
+            else
+                artifacts[path].reasons[idx].locs ~= to!int(r.line);
         }
-        else
-            artifacts[path].reasons[idx].locs ~= to!int(r.line);
-    }
-    auto accum = artifacts.values();
-    accum.sort!((a,b) => a.mod < b.mod || (a.mod == b.mod && a.id < b.id));
+        auto accum = artifacts.values();
+        accum.sort!((a,b) => a.mod < b.mod || (a.mod == b.mod && a.id < b.id));
 
-    string linkTemplate =
-`[%s/%s/blob/%s/%s.d#L%s %d] `;
-    writeln(`
-{| class="wikitable"
-! Module
-! Artifact
-! Reason
-! Possible Fix(es)
-|-`);
-    stderr.writefln("Total number of GC-happy artifacts: %s.", accum.length);
-    int attached = 0;
-    foreach(ref art; accum){
-        string[] comments;
-        foreach(i, t; talk)
-            if(t.mod == art.mod && art.id == t.artifact){
-                comments ~= t.comment;                
+        string linkTemplate =
+    `[%s/%s/blob/%s/%s.d#L%s %d] `;
+        writeln(`
+    {| class="wikitable"
+    ! Module
+    ! Artifact
+    ! Reason
+    ! Possible Fix(es)
+    |-`);
+        stderr.writefln("Total number of GC-happy artifacts: %s.", accum.length);
+        int attached = 0;
+        foreach(ref art; accum){
+            string[] comments;
+            foreach(i, t; talk)
+                if(t.mod == art.mod && art.id == t.artifact){
+                    comments ~= t.comment;                
+                }
+            if(comments.length){
+                attached += comments.length;
+                art.comment = comments.sort().uniq().join(";");
             }
-        if(comments.length){
-            attached += comments.length;
-            art.comment = comments.sort().uniq().join(";");
-        }
-    }
-    if(talk.length) //  talk.gchunt file was loaded
-        stderr.writefln("Successfully attached %d comments.", attached);
-    int filtered = 0;
-    foreach(art; accum){
-        if(blacklist.canFind!(black => black.match(art.mod, art.id))){
-            filtered++;
-            continue; // skip over if matches blacklist
         }
-        art.reasons.sort!((a,b) => a.reason < b.reason);
-        string reason = art.reasons.map!((r){
-            string links = r.reason~":";
-            foreach(i, loc; r.locs){
-                links ~= format(linkTemplate, gitHost, gitRepo, gitHash, 
-                    art.mod.replace(".","/"), loc, i+1);
+        if(talk.length) //  talk.gchunt file was loaded
+            stderr.writefln("Successfully attached %d comments.", attached);
+        int filtered = 0;
+        foreach(art; accum){
+            if(blacklist.canFind!(black => black.match(art.mod, art.id))){
+                filtered++;
+                continue; // skip over if matches blacklist
             }
-            return links;
-        }).join("\n\n");
-        writef("|%s\n|%s\n|%s\n| %s\n|-\n", art.mod, art.id, 
-            reason, art.comment);
+            art.reasons.sort!((a,b) => a.reason < b.reason);
+            string reason = art.reasons.map!((r){
+                string links = r.reason~":";
+                foreach(i, loc; r.locs){
+                    links ~= format(linkTemplate, gitHost, gitRepo, gitHash, 
+                        art.mod.replace(".","/"), loc, i+1);
+                }
+                return links;
+            }).join("\n\n");
+            writef("|%s\n|%s\n|%s\n| %s\n|-\n", art.mod, art.id, 
+                reason, art.comment);
+        }
+        writeln("|}");
+        if(blacklist.length)
+            stderr.writefln("Filtered %d artifacts.", filtered);     
     }
-    writeln("|}");
-    if(blacklist.length)
-        stderr.writefln("Filtered %d artifacts.", filtered);
+
+    
 }