Test File Formats

Test Configuration

Schema

configuration:
    type: object
    required:
        - task_configurations
        - categories
    properties:
        task_configurations:
            type: array
            items:
                $ref: '#/task_configuration'
        categories:
            type: array
            items:
                $ref: '#/category'
            
task_configuration:
    type: object
    required:
        - type
        - enabled
    properties:
        type:
            type: string
            enum: [similarity, analogy, neighborhood, outlier_detection]
        enabled:
            type: boolean            

category:
    type: object
    required:
        - name
        - enabled
        - tasks
        - categories
    properties:
        name:
            type: string
        enabled:
            type: boolean
        tasks:
            type: array
            items:
                $ref: '#/task'
        categories:
            type: array
            items:
                $ref: '#/category'

task:
    type: object
    required:
        - name
        - type
        - metric
        - test_set
    properties:
        name:
            type: string
        type:
            type: string
            enum: [similarity, analogy, neighborhood, outlier_detection]
        metric:
            type: string
            enum: [cosine, euclidean]
        test_set:
            type: string
            description: Path to local test-set file.

Example

configuration:
    task_configurations:
        - task_configuration: 
            type: similarity
            enabled: true
        - task_configuration:
            type: analogy
            enabled: true
        - task_configuration:
            type: neighborhood
            enabled: false
        - task_configuration:
            type: outlier_detection
            enabled: true
    categories:
        - category:
            name: {{category_name_#1}}
            enabled: true
            tasks:
                - task:
                    name: {{similarity_task_name_#1}}
                    type: similarity
                    metric: cosine
                    test_set: {{path_to_test_set_#1}}
                - task:
                    name: {{similarity_task_name_#2}}
                    type: similarity
                    metric: euclidean
                    test_set: {{path_to_test_set_#2}}
                - task:
                    name: {{analogy_task_name_#1}}
                    type: analogy
                    metric: cosine
                    test_set: {{path_to_test_set_#3}}
            categories:
                - category:
                    name: {{sub_category_name_#1}}
                    enabled: false
                    tasks:
                        - task:
                            name: {{neighborhood_task_name_#1}}
                            type: neighborhood
                            metric: euclidean
                            test_set: {{path_to_test_set_#4}}
                    categories:
                - category:
                    name: {{sub_category_name_#2}}
                    enabled: true
                    tasks:
                        - task:
                            name: {{outlier_detection_task_name_#1}}
                            type: outlier_detection
                            metric: cosine
                            test_set: {{path_to_test_set_#5}}
                    categories:
        - category:
            name: {{category_name_#2}}
            enabled: false
            tasks:
            categories:
                - category:
                    name: {{sub_category_name_#3}}
                    enabled: true
                    tasks:
                        - task:
                            name: {{similarity_task_name_#3}}
                            type: similarity
                            metric: euclidean
                            test_set: {{path_to_test_set_#6}}
                    categories:

Test Sets

Similarity

Schema:

a [:string], b [:string], is_similar [:boolean]

Example:

cake, muffin, true
car, tree, false

Analogy

Schema:

a [:string], b [:string]

Example:

Berlin, Germany
Paris, France

Neighborhood

Schema:

a [:string], group_id [:int], is_similar [:boolean]

Example:

cake, 1, true
muffing, 1, true
car, 2, false
tree, 2, false

Outlier Detection

Schema:

a [:string], group_id [:int], is_outlier [:boolean]

Example:

cake, 1, false
muffin, 1, false
car, 1, true