split stat in 2 separate crons

.circleci/config.yml

@@ -33,7 +33,7 @@ jobs:
           no_output_timeout: 60m
           command: opam exec -- make lang
 
-  stat:
+  stat1:
     docker:
       - image: returntocorp/ocaml:ubuntu
 
@@ -53,7 +53,32 @@ jobs:
       - run:
           name: run parsing stats
           no_output_timeout: 180m
-          command: opam exec -- make stat &> lang/stat.logs
+          command: opam exec -- make stat1
+      - run:
+          name: upload parsing stats
+          command: python3 ./scripts/upload-stat lang/stat.txt
+      - store_artifacts:
+          path: lang/stat.txt
+
+  # Mostly a copy-paste of stat1 but with make stat2
+  stat2:
+    docker:
+      - image: returntocorp/ocaml:ubuntu
+    resource_class: xlarge
+    working_directory: ~/ocaml-tree-sitter
+    steps:
+      - checkout
+      - run: git submodule update --init --recursive --depth 1
+      - run:
+          name: install nodejs and cargo
+          command: ./core/scripts/setup-node
+      - run: opam exec -- make setup
+      - run: opam exec -- make
+      - run: opam exec -- make install
+      - run:
+          name: run parsing stats
+          no_output_timeout: 180m
+          command: opam exec -- make stat2
       - run:
           name: upload parsing stats
           command: python3 ./scripts/upload-stat lang/stat.txt
@@ -68,7 +93,7 @@ workflows:
     jobs:
       - build
 
-  scheduled-stats:
+  scheduled-stats1:
     triggers:
       - schedule:
           # Run at 00:00 every day, UTC.
@@ -78,7 +103,24 @@ workflows:
               only:
                 - main
     jobs:
-      - stat:
+      - stat1:
+          # Run only on these branches (each pushing different images)
+          filters:
+            branches:
+              only:
+                - main
+
+  scheduled-stats2:
+    triggers:
+      - schedule:
+          # Run at 06:00 every day, UTC.
+          cron: "6 0 * * *"
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - stat2:
           # Run only on these branches (each pushing different images)
           filters:
             branches:
@@ -89,7 +131,9 @@ workflows:
   # to a branch named 'stats'.
   stats-on-commit:
     jobs:
-      - stat:
+      # This just trigger stat1. We could have another special branch to
+      # trigger stat2 on-commit.
+      - stat1:
           filters:
             branches:
               only:

Makefile

@@ -49,6 +49,17 @@ stat:
 	$(MAKE) -C lang
 	$(MAKE) -C lang stat
 
+# ugly: but split make stat in 2 to pass the 3h CI limit
+.PHONY: stat1
+stat1:
+	$(MAKE) -C lang
+	$(MAKE) -C lang stat1
+
+.PHONY: stat2
+stat2:
+	$(MAKE) -C lang
+	$(MAKE) -C lang stat2
+
 .PHONY: install
 install:
 	$(MAKE) -C core install

lang/Makefile

@@ -87,9 +87,13 @@ SUPPORTED_DIALECTS = \
 
 # Languages which are set up to run parsing stats. Ideally, this is all
 # the supported languages. See the 'stat' target.
-#
-STAT_LANGUAGES = \
+# update: we split the list in 2 (STAT_LANGUAGES1 and STAT_LANGUAGES2) because
+# running all languages in CI takes more than 3h which is the performance-plan limit.
+# alt: parse languages in parallel, like we do in semgrep/stats/parsing-stats/
+STAT_LANGUAGES1 = \
   bash \
+  c \
+  cpp \
   c-sharp \
   dockerfile \
   elixir \
@@ -98,7 +102,11 @@ STAT_LANGUAGES = \
   java \
   javascript \
   julia \
-  kotlin \
+  kotlin
+# Commented out due to hitting memory limits in CI:
+#  hack
+
+STAT_LANGUAGES2 = \
   lua \
   ocaml \
   php \
@@ -110,10 +118,8 @@ STAT_LANGUAGES = \
   solidity \
   tsx \
   typescript
-# Commented out due to hitting memory limits in CI:
-#  hack
-# Commented out to pass the 3h time limit in CI:
-#  c cpp sqlite
+
+STAT_LANGUAGES = $(STAT_LANGUAGES1) $(STAT_LANGUAGES2)
 
 # Build and test each language. Does not run parsing stats (see 'stat:').
 #
@@ -157,15 +163,27 @@ dry:
 	  ./release $$lang --dry-run; \
 	done
 
+.PHONY: stat
+stat:
+	$(MAKE) stat1
+	$(MAKE) stat2
+
 # Run parsing stats for each language.
 # Each needs a list of projects (projects.txt) and file extensions
 # (extensions.txt).
 #
-.PHONY: stat
-stat:
+.PHONY: stat1
+stat1:
 	set -eu; \
-	for lang in $(STAT_LANGUAGES); do $(MAKE) -C $$lang stat; done
-	../scripts/report-stat $(STAT_LANGUAGES) | tee stat.txt
+	for lang in $(STAT_LANGUAGES1); do $(MAKE) -C $$lang stat; done
+	../scripts/report-stat $(STAT_LANGUAGES1) | tee stat.txt
+	test -s stat.txt  # check that the pipeline above didn't fail
+
+.PHONY: stat2
+stat2:
+	set -eu; \
+	for lang in $(STAT_LANGUAGES2); do $(MAKE) -C $$lang stat; done
+	../scripts/report-stat $(STAT_LANGUAGES2) | tee stat.txt
 	test -s stat.txt  # check that the pipeline above didn't fail
 
 # Run stats on projects-priv.txt instead of projects.txt.