Skip to content

Commit e761f3e

Browse files
akarivakariv
committed
Initial commit
0 parents  commit e761f3e

9 files changed

+917
-0
lines changed

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
node_modules/
2+
coverage
3+
dist/
4+
.idea/

‎.travis.yml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
sudo: false
2+
language: node_js
3+
cache:
4+
directories:
5+
- node_modules
6+
branches:
7+
only:
8+
- master
9+
notifications:
10+
email: false
11+
node_js:
12+
- iojs
13+
before_install:
14+
- npm i -g npm@^2.0.0
15+
before_script:
16+
- npm prune
17+
script:
18+
- npm run test:single
19+
- npm run check-coverage
20+
- npm run build
21+
after_success:
22+
- npm run report-coverage
23+
- npm run semantic-release

‎openspending-types.csv

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
type,type,type,type,type,parent,labelFor,dataType,uniqueIdentifier
2+
functional-classification,cofog,division,code,full,,,string,y
3+
,,,,part,,,string,y
4+
,,,label,,,functional-classification:cofog:division:code,string,
5+
,,,description,,,,string,
6+
,,group,code,full,functional-classification:cofog:division:code,,string,y
7+
,,,,part,functional-classification:cofog:division:code,,string,y
8+
,,,label,,,functional-classification:cofog:group:code,string,
9+
,,,description,,,,string,
10+
,,class,code,full,functional-classification:cofog:group:code,,string,y
11+
,,,,part,functional-classification:cofog:group:code,,string,y
12+
,,,label,,,functional-classification:cofog:class:code,string,
13+
,,,description,,,,string,
14+
,,code,full,,,,string,y
15+
,,,part,,,,string,y
16+
,,label,,,,functional-classification:cofog:code,string,
17+
,,description,,,,,string,
18+
,generic,level1,code,full,,,string,y
19+
,,,,part,,,string,y
20+
,,,label,,,functional-classification:generic:level1:code,string,
21+
,,,description,,,,string,
22+
,,level2,code,full,functional-classification:generic:level1:code,,string,y
23+
,,,,part,functional-classification:generic:level1:code,,string,y
24+
,,,label,,,functional-classification:generic:level2:code,string,
25+
,,,description,,,,string,
26+
,,level3,code,full,functional-classification:generic:level2:code,,string,y
27+
,,,,part,functional-classification:generic:level2:code,,string,y
28+
,,,label,,,functional-classification:generic:level3:code,string,
29+
,,,description,,,,string,
30+
,,level4,code,full,functional-classification:generic:level3:code,,string,y
31+
,,,,part,functional-classification:generic:level3:code,,string,y
32+
,,,label,,,functional-classification:generic:level4:code,string,
33+
,,,description,,,,string,
34+
,,code,full,,,,string,y
35+
,,,part,,,,string,y
36+
,,label,,,,,string,
37+
,,description,,,,,string,
38+
administrative-classification,generic,level1,code,full,,,string,y
39+
,,,,part,,,string,y
40+
,,,label,,,administrative-classification:generic:level1:code,string,
41+
,,,description,,,,string,
42+
,,level2,code,full,administrative-classification:generic:level1:code,,string,y
43+
,,,,part,administrative-classification:generic:level1:code,,string,y
44+
,,,label,,,administrative-classification:generic:level2:code,string,
45+
,,,description,,,,string,
46+
,,level3,code,full,administrative-classification:generic:level2:code,,string,y
47+
,,,,part,administrative-classification:generic:level2:code,,string,y
48+
,,,label,,,administrative-classification:generic:level3:code,string,
49+
,,,description,,,,string,
50+
,,level4,code,full,administrative-classification:generic:level3:code,,string,y
51+
,,,,part,administrative-classification:generic:level3:code,,string,y
52+
,,,label,,,administrative-classification:generic:level4:code,string,
53+
,,,description,,,,string,
54+
,,code,full,,,,string,y
55+
,,,part,,,,string,y
56+
,,label,,,,administrative-classification:generic:code,string,
57+
,,description,,,,,string,
58+
economic-classification,gfsm,level1,code,,,,string,y
59+
,,,label,,,economic-classification:gfsm:level1:code,string,y
60+
,,,description,,,,string,
61+
,,level2,code,,economic-classification:gfsm:level1:code,,string,y
62+
,,,label,,,economic-classification:gfsm:level2:code,string,y
63+
,,,description,,,,string,
64+
,,level3,code,,economic-classification:gfsm:level2:code,,string,y
65+
,,,label,,,economic-classification:gfsm:level3:code,string,y
66+
,,,description,,,,string,
67+
,,level4,code,,economic-classification:gfsm:level3:code,,string,y
68+
,,,label,,,economic-classification:gfsm:level4:code,string,y
69+
,,,description,,,,string,
70+
,generic,level1,code,full,,,string,y
71+
,,,,part,,,string,y
72+
,,,label,,,economic-classification:generic:level1:code,string,
73+
,,,description,,,,string,
74+
,,level2,code,full,economic-classification:generic:level1:code,,string,y
75+
,,,,part,economic-classification:generic:level1:code,,string,y
76+
,,,label,,,economic-classification:generic:level2:code,string,
77+
,,,description,,,,string,
78+
,,level3,code,full,economic-classification:generic:level2:code,,string,y
79+
,,,,part,economic-classification:generic:level2:code,,string,y
80+
,,,label,,,economic-classification:generic:level3:code,string,
81+
,,,description,,,,string,
82+
,,level4,code,full,economic-classification:generic:level3:code,,string,y
83+
,,,,part,economic-classification:generic:level3:code,,string,y
84+
,,,label,,,economic-classification:generic:level4:code,string,
85+
,,,description,,,,string,
86+
,,code,full,,,,string,y
87+
,,,part,,,,string,y
88+
,,label,,,,,string,
89+
,,description,,,,,string,
90+
activity,generic,program,code,,,,string,y
91+
,,,label,,,activity:generic:program:code,string,
92+
,,project,code,,activity:generic:program:code,,string,y
93+
,,,label,,,activity:generic:project:code,string,
94+
,,contract,code,,activity:generic:project:code,,string,y
95+
,,,label,,,activity:generic:contract:code,string,
96+
value,,,,,,,number,
97+
administrator,generic,id,,,,,string,y
98+
,,name,,,,,string,
99+
supplier,generic,id,,,,,string,y
100+
,,name,,,,,string,
101+
procurer,generic,id,,,,,string,y
102+
,,name,,,,,string,
103+
recipient,generic,id,,,,,string,y
104+
,,name,,,,,string,
105+
geo-source,source,code,,,,,string,y
106+
,,title,,,,,string,
107+
,,codeList,,,,,string,y
108+
,target,code,,,,,string,y
109+
,,title,,,,,string,
110+
,,codeList,,,,,string,y
111+
,generic,code,,,,,string,y
112+
,,title,,,,,string,
113+
,,codeList,,,,,string,y
114+
date,fiscal-year,,,,,,integer,y
115+
,generic,,,,,,date,y
116+
direction,,,,,,,string,y
117+
phase,,,,,,,string,y
118+
invoice-id,,,,,,,string,y
119+
budget-line-id,,,,,,,string,y

‎os-types.iml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<module type="WEB_MODULE" version="4">
3+
<component name="NewModuleRootManager" inherit-compiler-output="true">
4+
<exclude-output />
5+
<content url="file://$MODULE_DIR$" />
6+
<orderEntry type="inheritedJdk" />
7+
<orderEntry type="sourceFolder" forTests="false" />
8+
<orderEntry type="library" name="os-types node_modules" level="project" />
9+
</component>
10+
</module>

‎package.json

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{
2+
"name": "os-types",
3+
"version": "0.0.0-semantically-released",
4+
"description": "Fiscal Data Type Handling",
5+
"main": "dist/index.js",
6+
"scripts": {
7+
"prebuild": "rm -rf dist && mkdir dist",
8+
"build": "babel src/index.js -o dist/index.js",
9+
"postbuild": "cp src/os-types.json dist/",
10+
"commit": "git-cz",
11+
"check-coverage": "istanbul check-coverage --statements 90 --branches 90 --functions 90 --lines 90",
12+
"report-coverage": "cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js",
13+
"start": "npm run test",
14+
"test": "mocha src/index.test.js -w --compilers js:babel/register",
15+
"test:single": "istanbul cover -x *.test.js _mocha -- -R spec src/index.test.js --compilers js:babel/register",
16+
"semantic-release": "semantic-release pre && npm publish && semantic-release post"
17+
},
18+
"repository": {
19+
"type": "git",
20+
"url": "https://github.com/kentcdodds/starwars-names.git"
21+
},
22+
"keywords": [],
23+
"author": "Adam Kariv <[email protected]>",
24+
"license": "MIT",
25+
"bugs": {
26+
"url": "https://github.com/openspending/os-types/issues"
27+
},
28+
"homepage": "https://github.com/openspending/os-types#readme",
29+
"dependencies": {
30+
"coveralls": "^2.11.9",
31+
"lodash": "4.10.0",
32+
"lodash-addons": "^0.3.1"
33+
},
34+
"devDependencies": {
35+
"babel": "5.8.21",
36+
"chai": "3.3.0",
37+
"codecov.io": "0.1.6",
38+
"commitizen": "1.0.5",
39+
"cz-conventional-changelog": "1.1.2",
40+
"ghooks": "0.3.2",
41+
"istanbul": "git+https://github.com/gotwarlost/istanbul#source-map",
42+
"mocha": "2.2.5",
43+
"semantic-release": "4.3.5"
44+
},
45+
"czConfig": {
46+
"path": "node_modules/cz-conventional-changelog"
47+
},
48+
"config": {
49+
"ghooks": {
50+
"pre-commit": "npm run test:single && npm run check-coverage"
51+
}
52+
}
53+
}

‎parse-types-csv.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import csv
2+
import json
3+
4+
ident = lambda x: x
5+
parsers = {
6+
'parent': ident,
7+
'labelFor': ident,
8+
'dataType': ident,
9+
'uniqueIdentifier': lambda x: x=='y',
10+
}
11+
12+
if __name__ == "__main__":
13+
infile = iter(csv.reader(open('openspending-types.csv')))
14+
header = next(infile)
15+
prev_types = []
16+
types = {}
17+
for row in infile:
18+
idx = 0
19+
type_name = None
20+
for hdr, val in zip(header,row):
21+
val = val.strip()
22+
if len(val)>0:
23+
if hdr == 'type':
24+
if len(prev_types) == idx:
25+
prev_types.append(val)
26+
elif len(prev_types) < idx:
27+
print(prev_types, idx, row)
28+
assert(False);
29+
else:
30+
prev_types[idx] = val
31+
prev_types = prev_types[:idx+1]
32+
else:
33+
if type_name is None:
34+
type_name = ':'.join(prev_types)
35+
types.setdefault(type_name, {})[hdr] = parsers[hdr](val)
36+
idx += 1
37+
json.dump(types, open('src/os-types.json','w'), indent=2, sort_keys=True)

‎src/index.js

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
'use strict';
2+
var os_types = require('./os-types.json');
3+
var _ = require('lodash-addons');
4+
5+
class TypeProcessor {
6+
7+
constructor() {
8+
this.types = os_types;
9+
}
10+
11+
getAllTypes() {
12+
return Object.keys(this.types);
13+
};
14+
15+
_checkInput(fields) {
16+
// Make sure we got an array...
17+
var valid = _.isArray(fields);
18+
// ... of objects ...
19+
valid = valid &&
20+
_.every(fields, (f) => {
21+
return _.isObject(f);
22+
});
23+
// ... with all the mandatory properties ...
24+
valid = valid &&
25+
_.every(fields, (f) => {
26+
return _.hasIn(f, 'title') && _.hasIn(f, 'type');
27+
});
28+
// ... and no unknown properties ...
29+
valid = valid &&
30+
_.every(fields, (f) => {
31+
return _.difference(_.keys(f), ['title', 'type', 'format', 'data']).length == 0;
32+
});
33+
// ... and all types are valid ...
34+
valid = valid &&
35+
_.every(fields, (f) => {
36+
return _.hasIn(this.types, f.type);
37+
});
38+
return valid;
39+
}
40+
41+
_titleToName(title, type) {
42+
var slugRe = new RegExp('[a-zA-Z0-9]+','g');
43+
var vowelsRe = new RegExp('[aeiou]+','g');
44+
var slugs = _.deburr(title).match(slugRe);
45+
if ( slugs == null || slugs.length == 0 ) {
46+
slugs = _.join(type.split(vowelsRe),'').match(slugRe);
47+
}
48+
var name = _.join(slugs, '_');
49+
if ( this.allNames.indexOf(name) >= 0 ) {
50+
let i = 2;
51+
while ( true ) {
52+
let attempt = name + '_' + i;
53+
if ( this.allNames.indexOf(attempt) < 0 ) {
54+
name = attempt;
55+
break;
56+
}
57+
i+=1;
58+
}
59+
}
60+
this.allNames.push(name)
61+
return name;
62+
}
63+
64+
fieldsToModel(fields) {
65+
// Detect invalid data
66+
if ( !this._checkInput(fields) ) {
67+
return null;
68+
}
69+
// Modelling
70+
var dimensions = {};
71+
var measures = {};
72+
var model = { dimensions, measures };
73+
var schema = {};
74+
this.allNames = [];
75+
_.forEach(fields, (f) => {
76+
var osType = this.types[f.type];
77+
f.name = this._titleToName(f.title, f.type);
78+
schema[f.title] = {
79+
title: f.title,
80+
name: f.name,
81+
type: osType.dataType,
82+
format: osType.format || f.format || 'default'
83+
};
84+
85+
var conceptType = _.split(f.type,':')[0];
86+
if ( conceptType == 'value' ) {
87+
// Measure
88+
var measure = {
89+
source: f.name,
90+
resource: f.resource
91+
};
92+
measures[f.name] = measure;
93+
} else {
94+
let dimension;
95+
if ( _.hasIn(dimensions, conceptType) ) {
96+
dimension = dimensions[conceptType];
97+
} else {
98+
dimension = {
99+
dimensionType: osType.dimensionType,
100+
primaryKey: [],
101+
attributes: {}
102+
};
103+
dimensions[conceptType] = dimension;
104+
}
105+
var attribute = {
106+
source: f.name,
107+
resource: f.resource
108+
};
109+
dimension.attributes[f.name] = attribute;
110+
if ( osType.uniqueIdentifier ) {
111+
dimension.primaryKey.push(f.name);
112+
}
113+
}
114+
});
115+
var ret = {model, schema};
116+
console.log(JSON.stringify(ret));
117+
return ret;
118+
}
119+
}
120+
module.exports = TypeProcessor;

0 commit comments

Comments
 (0)