update for exporting to hydroshare

odm2admin/management/commands/create_sqlite_export.py

@@ -13,8 +13,8 @@
 from odm2admin.models import Dataloggerfiles
 from templatesAndSettings.settings import exportdb
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "templatesAndSettings.settings.exportdb")
-
+#os.environ.setdefault("DJANGO_SETTINGS_MODULE", "templatesAndSettings.settings.exportdb")
+os.environ['DJANGO_SETTINGS_MODULE'] = "templatesAndSettings.settings.exportdb"
 __author__ = 'leonmi'
 
 
@@ -27,13 +27,24 @@ class Command(BaseCommand):
     def add_arguments(self, parser):
         parser.add_argument('jsonfile1', nargs=1, type=str)
         parser.add_argument('jsonfile2', nargs=1, type=str)
-
+        parser.add_argument('dbfile', nargs=1, type=str)
 
     def handle(self, *args, **options):  # (f,fileid, databeginson,columnheaderson, cmd):
         # cmdline = bool(options['cmdline'][0])
-        jsonfile1 = str(options['jsonfile1'][0])
-        jsonfile2 = str(options['jsonfile2'][0])
-        print(jsonfile1,jsonfile2)
-        print(connection.settings_dict['NAME'])
-        management.call_command('loaddata',jsonfile1, settings=exportdb)  # ,database='export'
-        management.call_command('loaddata',jsonfile2, settings=exportdb)
+        try:
+            jsonfile1 = str(options['jsonfile1'][0])
+            jsonfile2 = str(options['jsonfile2'][0])
+            dbfile = str(options['dbfile'][0])
+            self.stdout.write('start data load')
+            # print('start data load')
+            #f = open('/home/azureadmin/webapps/logs/logfile.txt', 'rw')
+            #f.write('start load data')
+            #f.write(jsonfile1,jsonfile2)
+            exportdb.DATABASES['default']['NAME'] = dbfile
+            # print(connection.settings_dict['NAME'])
+            #f.close()
+            management.call_command('loaddata',jsonfile1)  # ,database='export'
+            management.call_command('loaddata',jsonfile2)
+            # print('end data load')
+        except Exception, e:
+            return e

odm2admin/views.py

@@ -3,6 +3,8 @@
 import json
 import time
 import sys
+import os
+import subprocess32 as subprocess
 from datetime import datetime
 from datetime import timedelta
 from time import mktime
@@ -30,6 +32,7 @@
 # from oauth2_provider.views.generic import ProtectedResourceView
 from django.http import HttpResponse
 from django.forms.models import model_to_dict
+from django.utils.crypto import get_random_string
 # from django.contrib.gis.geos import GEOSGeometry
 # import hs_restclient as hs_r
 from hs_restclient import HydroShare, HydroShareAuthOAuth2
@@ -1667,16 +1670,22 @@ def createODM2SQLiteFile(request):
     #management.call_command('dump_object', 'odm2admin.Timeseriesresults', 17160, 17162, kitchensink=True)
     sysout = sys.stdout
     loc = settings.FIXTURE_DIR
-    print(myresultSeriesExport.first())
-    sys.stdout = open(loc+ 'tmp.json', 'w')
-    tmploc1 = loc+ 'tmp.json'
+    # print(myresultSeriesExport.first())
+    random_string = get_random_string(length=5)
+    tmpfixture1 = 'tmp'  + '.json' #+ random_string
+    sys.stdout = open(loc+ tmpfixture1, 'w')
+    tmploc1 = loc+ tmpfixture1
     management.call_command('dump_object', 'odm2admin.Timeseriesresultvalues', myresultSeriesExport.first().valueid, kitchensink=True)
     sys.stdout.close()
     #jsonfile = open(loc+ 'tmp2.json', 'w')
     # i=0
     values = myresultSeriesExport.values_list('valueid', flat=True)
-    sys.stdout = open(loc + 'tmp2.json', 'w')
-    tmploc2 = loc+ 'tmp2.json'
+    random_string = get_random_string(length=5)
+    # add random string back later
+    tmpfixture2 = 'tmp'  + '.json' # + random_string
+    sys.stdout = open(loc + tmpfixture2, 'w')
+    # sys.stdout = open(loc + 'tmp2.json, 'w')
+    tmploc2 = loc+ tmpfixture1
     sys.stdout.write(serializers.serialize("json", myresultSeriesExport[1:], indent=4,use_natural_foreign_keys=False,use_natural_primary_keys=False))
     sys.stdout.close()
     sys.stdout = sysout
@@ -1702,8 +1711,33 @@ def createODM2SQLiteFile(request):
     #management.call_command('loaddata',
     #                        tmploc2,database=database)
     #export_data.send(sender= Timeseriesresultvalues,tmploc1=tmploc1,tmploc2=tmploc2)
-    management.call_command('create_sqlite_export',tmploc1,tmploc2, settings=exportdb)
+    #management.call_command('create_sqlite_export',tmploc1,tmploc2, settings=exportdb)
+    # call('../')
+    # print(tmploc1)
+    # print(tmploc2)
+    dbfilepath = exportdb.DATABASES['default']['NAME']
+    path = os.path.dirname(dbfilepath)
+    dbfile = os.path.basename(dbfilepath)
+    dbfilename = os.path.splitext(dbfile)[0]
+    random_string = get_random_string(length=5)
+    dbfile2 = path +"/" + dbfilename +  random_string + ".db"
+    #command = ['python',  '/home/azureadmin/webapps/ODM2-AdminLCZO/manageexport.py', 'create_sqlite_export', tmploc1, tmploc2]
+    command = 'cp ' + dbfilepath + ' ' + str(dbfile2)
+    # print(command)
+    response = subprocess.check_call(command,shell=True)
+    #write an extra settings file instead - have it contain just DATABASES; remove databases from exportdb.py and import new file. 2
+    exportdb.DATABASES['default']['NAME'] = dbfile2
+    command = settings.BASE_DIR + '/scripts/create_sqlite_file.sh '+ dbfile2 + ' %>> ' + settings.BASE_DIR +'/logging/sqlite_export.log'
+    # print(command)
+    response = subprocess.check_call(command,shell=True) #
+    # print("response")
+    # print(response)
+    # print(exportdb.DATABASES['default']['NAME'])
     return myresultSeriesExport
+    # outfile = loc +'tmp2.json'
+    # print(outfile)
+    # with open(outfile, 'w') as jsonfile:
+    #    json.dump(data, jsonfile)
     #outfile = loc +'tmp2.json'
     #print(outfile)
     #with open(outfile, 'w') as jsonfile:
@@ -1763,6 +1797,7 @@ def export_to_hydroshare(request):
     keywords = ('test', 'test 2')
     rtype = 'GenericResource'
     fpath = exportdb.DATABASES['default']['NAME']
+    # print(fpath)
     #metadata = '[{"coverage":{"type":"period", "value":{"start":"'+entered_start_date +'", "end":"'+ entered_end_date +'"}}}, {"creator":{"name":"Miguel Leon"}}]'
     metadata = '[{"coverage":{"type":"period", "value":{"start":"03/26/2017", "end":"04/25/2017"}}}, {"creator":{"name":"Miguel Leon"}}]'
     extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'

requirements.txt

@@ -5,11 +5,17 @@ django-apptemplates
 django-daterange-filter
 django-import-export
 django-jquery
+# django-kronos
 django==1.10.5
 # Pinned to 0.2.7 to preserve page style.
 djangocms-admin-style==0.2.7
 psycopg2
 requests
 django-recaptcha
+# django-redis
 pyyaml
-xlrd
+xlrd
+hs_restclient
+django-fixture-magic
+social-auth-app-django
+subprocess32

templatesAndSettings/scripts/create_sqlite_file.sh

@@ -0,0 +1,2 @@
+/home/azureadmin/miniconda2/envs/odm2adminenv2/bin/python /home/azureadmin/webapps/ODM2-AdminLCZO/manageexport.py create_sqlite_export /home/azureadmin/webapps/ODM2-AdminLCZO/templatesAndSettings/fixtures/tmp.json /home/azureadmin/webapps/ODM2-AdminLCZO/templatesAndSettings/fixtures/tmp2.json $1
+