Improve network analysis

Author: pkiraly

export-network.sh

@@ -100,7 +100,7 @@ do
     done
   done
 
-  for TYPE in "density" "clustering-coefficient"; do 
+  for TYPE in "density" "clustering-coefficient" "triCountGraph-stat"; do 
     # PREFIX=density
     if [[ $TAG == "all" && -e $OUTPUT_DIR/network-scores-${TYPE}.csv ]]; then
       rm $OUTPUT_DIR/network-scores-${TYPE}.csv
@@ -111,6 +111,6 @@ do
         | grep -v -P "^\d+,average-clustering-coefficient$" \
         | sed 's/all,average-clustering-coefficient/tag,average-clustering-coefficient/' \
         >> $OUTPUT_DIR/network-scores-${TYPE}.csv
-    # rm $OUTPUT_DIR/network-scores-$TAG-${TYPE}.csv
+    rm $OUTPUT_DIR/network-scores-$TAG-${TYPE}.csv
   done
 done < $INPUT_DIR/network-by-concepts-tags.csv

scala/src/main/scala/de/gwdg/metadataqa/marc/spark/Network.scala

@@ -181,10 +181,16 @@ object Network {
     log.info("STEP 5: clustering coefficients")
 
     val triCountGraph = graph.triangleCount()
-    // triCountGraph.vertices.map(x => x._2).stats()
-    val tricountDF = triCountGraph.vertices.toDF("id", "count")
+    val triCountGraphstat = triCountGraph.vertices
+      .map(x => x._2)
+      .toDF("x")
+      .select("x")
+      .summary().toDF("statistic", "value")
+    this.write("network-scores" + suffix + "-triCountGraph-stat", triCountGraphstat)
+    // val tricountDF = triCountGraph.vertices.toDF("id", "count")
 
     // var degreesRDD = graph.degrees.cache()
+    /*
     val maxTrisGraph = graph.degrees.mapValues(d => d * (d - 1) / 2.0)
     val maxTrisDF = maxTrisGraph.toDF("id", "theoreticalMax")
 
@@ -199,6 +205,7 @@ object Network {
 
     val dataDF = Seq(arageClusteringCoefficient).toDF("average-clustering-coefficient")
     this.write("network-scores" + suffix + "-clustering-coefficient", dataDF)
+     */
   }
 
   def write(file: String, df: DataFrame): Unit = {