Add more large data (#124)

Author: srmukher

apps/plotly_examples/python-scripts/generate_plotly_schema.py

@@ -69,18 +69,43 @@ def generate_visualization_schemas():
             id = call_llm_plotly_schema(json.dumps(scenario_data), id)
 
 def generate_detailed_visualization_schemas():
+    print('Generating detailed schemas')
     scenario_dir = 'generated_schema'
     if not os.path.exists(scenario_dir):
         print("Populate the schemas first by calling generate_visualization_schemas()")
-    id = 253
+    id = 303
     sample_size = min(25, len(os.listdir(scenario_dir)))
-    random_files = random.sample(os.listdir(scenario_dir), sample_size)
+    chart_types = {}
+    for file_name in os.listdir(scenario_dir):
+        json_data = read_json_file(os.path.join(scenario_dir, file_name))
+        chart_type = json_data['data'][0]['type']
+        if chart_type not in chart_types:
+            chart_types[chart_type] = []
+        chart_types[chart_type].append(file_name)
+        
+    min_samples_per_chart_type = sample_size//len(chart_types)
+    random_files = []
+    for chart_type, files in chart_types.items():
+        random_files.extend(random.sample(files, min(min_samples_per_chart_type, len(files))))
+    
+    # keeping extra buffer data in case any one fails to generate
+    buffer_data = 50
+    if len(random_files) < sample_size + buffer_data:
+        remaining_files = [file for file in os.listdir(scenario_dir) if file not in random_files]
+        if len(remaining_files) > 0:
+            random_files.extend(random.sample(remaining_files, ((sample_size+buffer_data)-len(random_files))))
+
+    count = 0
     for file_name in random_files:
+        if count == sample_size:
+            break
         json_data = read_json_file(os.path.join(scenario_dir, file_name))
         curr_id = json_data['id']
         file_name_prefix = file_name.split('.')[0]
         suffix = file_name_prefix.split(str(curr_id))[1]
-        id = call_llm_detailed_plotly_schema(json.dumps(json_data), id, suffix)
+        id, isSuccess = call_llm_detailed_plotly_schema(json.dumps(json_data), id, suffix)
+        if isSuccess:
+            count = count + 1
 
 def generate_locale_visualization_schemas():
     if not os.path.exists(scenario_dir):
@@ -303,7 +328,7 @@ def call_llm_detailed_plotly_schema(scenario: str, id: int, suffix: str):
     # call only if the file does not exist
     if os.path.exists(f'generated_schema_detailed/data_{id}{suffix}.json'):
         print(f"Skipping {id}_{suffix}")
-        return id+1
+        return id+1, False
 
     # in case text_output is not a valid json, it will retry 3 times
     retry_count=0
@@ -318,7 +343,7 @@ def call_llm_detailed_plotly_schema(scenario: str, id: int, suffix: str):
 
     if retry_count == 3:
         print("Failed to generate schema")
-        return id
+        return id, False
 
     output_dir = 'generated_schema_detailed'
     os.makedirs(output_dir, exist_ok=True)
@@ -329,7 +354,7 @@ def call_llm_detailed_plotly_schema(scenario: str, id: int, suffix: str):
         json.dump(data, file, indent=4)
         id=id+1
 
-    return id
+    return id, True
 
 def get_chart_type_from_image():
     directory_path = os.path.join('..', 'tests', 'Plotly.spec.ts-snapshots')
@@ -390,10 +415,10 @@ def get_chart_type_from_image():
 # generate_visualization_schemas()
 
 # Generate detailed schemas
-# generate_detailed_visualization_schemas()
+generate_detailed_visualization_schemas()
 
 # Generate locale based schemas
 # generate_locale_visualization_schemas()
 
 # Generate chart types from screenshots taken by Playwright
-get_chart_type_from_image()
+# get_chart_type_from_image()

apps/plotly_examples/src/components/DeclarativeChart.tsx

@@ -41,9 +41,9 @@ type DataType =
   | 'localization';
 
 const dataTypeRanges = {
-  'general': { min: 1, max: 252 },
-  'largeData': { min: 253, max: 277 },
-  'localization': { min: 278, max: 302 }
+  'general': [{ min: 1, max: 252 }],
+  'largeData': [{ min: 253, max: 277 }, { min: 303, max: 332 }],
+  'localization': [{ min: 278, max: 302 }]
 };
 
 // Use require.context to load all JSON files from the split_data folder
@@ -128,8 +128,8 @@ const DeclarativeChartBasicExample: React.FC<IDeclarativeChartProps> = () => {
       .filter((data) => {
         const schemaId = parseInt((data.schema as { id: string }).id, 10);
         return selectedDataTypes.includes('All') || selectedDataTypes.some(dataType => {
-          const range = dataTypeRanges[dataType as keyof typeof dataTypeRanges];
-          return schemaId >= range.min && schemaId <= range.max;
+          if (dataType === 'All') return true;
+          return dataTypeRanges[dataType].some(range => schemaId >= range.min && schemaId <= range.max);
         });
       })
       .filter((data) => {

apps/plotly_examples/src/components/aggregated_chart_types.json

@@ -300,5 +300,35 @@
   "299": "GroupedVerticalBar",
   "300": "GroupedVerticalBar",
   "301": "GroupedVerticalBar",
-  "302": "GroupedVerticalBar"
+  "302": "GroupedVerticalBar",
+  "303": "GroupedVerticalBar",
+  "304": "VerticalBar",
+  "305": "Pie",
+  "306": "Donut",
+  "307": "Line",
+  "308": "Line",
+  "309": "Line",
+  "310": "Line",
+  "311": "Heatmap",
+  "312": "Others",
+  "313": "Others",
+  "314": "Others",
+  "315": "Others",
+  "316": "Others",
+  "317": "GroupedVerticalBar",
+  "318": "Line",
+  "319": "GroupedVerticalBar",
+  "320": "Line",
+  "321": "GroupedVerticalBar",
+  "322": "Others",
+  "323": "Donut",
+  "324": "GroupedVerticalBar",
+  "325": "Pie",
+  "326": "VerticalStackedBar",
+  "327": "VerticalStackedBar",
+  "328": "Area",
+  "329": "Area",
+  "330": "HorizontalBarWithAxis",
+  "331": "HorizontalBarWithAxis",
+  "332": "Sankey"
 }