On-demand UDL test support, synch improvements

cls/Port/Configuration.cls

@@ -54,13 +54,14 @@ ClassMethod IsAutoExportXMLEnabled() As %Status
 /// Relative to it's project. Sets the path where test cases should be exported.
 ClassMethod SetTestPath(newPath As %String = "")
 {
-
   set ^Port.Configuration("test.path") = $select(newPath = "" : "tests/cls", 1: newPath)
 }
 
 /// Retrives the resolved path where tests are being exported.
 ClassMethod GetTestPath() As %String
 {
+  set isUsingUDL = (..GetTestFormat() = "UDL")
+  if isUsingUDL return "cls"
   return ..RewritePlaceHolders($get(^Port.Configuration("test.path")),,$lb("{NAMESPACE}","{PROJECT}","{USERNAME}"))
 }
 
@@ -91,7 +92,6 @@ ClassMethod SetTestClassPrefix(newPrefix As %String = "")
 /// Gets the associated test prefix.
 ClassMethod GetTestClassPrefix() As %String
 {
-
   return $get(^Port.Configuration("test.prefix"))
 }
 
@@ -147,24 +147,28 @@ ClassMethod GetExtendedHooksImplementer() As %String
 ClassMethod SetRoutineInputTranslateTable(charset As %String) As %Status
 {
   set ^Port.Configuration("io.routine.in") = charset
+  return $$$OK
 }
 
 /// Defines the encoding used to save the exported files originated from routines, this includes class items. Pass RAW to preserve the content as-is.
-ClassMethod SetRoutineOutoutTranslateTable(charset As %String) As %Status
+ClassMethod SetRoutineOutputTranslateTable(charset As %String) As %Status
 {
-  set ^Port.Configuration("io.public.out") = charset
+  set ^Port.Configuration("io.routine.out") = charset
+  return $$$OK
 }
 
 /// Defines the encoding used to save CSP items. Pass RAW to preserve the content as-is.
 ClassMethod SetPublicFileInputTranslateTable(charset As %String) As %Status
 {
   set ^Port.Configuration("io.public.in") = charset
+  return $$$OK
 }
 
 /// Defines the encoding used to save public files originated from CSP items. Pass RAW to preserve the content as-is.
 ClassMethod SetPublicFileOutputTranslateTable(charset As %String) As %Status
 {
   set ^Port.Configuration("io.public.out") = charset
+  return $$$OK
 }
 
 /// Retrieves the current configured routine translate table for inputs. Defaults to UTF-8.
@@ -176,7 +180,7 @@ ClassMethod GetRoutineInputTranslateTable(charset As %String) As %Status
 /// Retrieves the current configured routine translate table for outputs. Defaults to UTF-8.
 ClassMethod GetRoutineOutputTranslateTable(charset As %String) As %Status
 {
-  return $get(^Port.Configuration("io.public.out"), "UTF8")
+  return $get(^Port.Configuration("io.routine.out"), "UTF8")
 }
 
 /// Retrieves the current configured routine translate table for outputs. Defaults to UTF-8.
@@ -191,6 +195,18 @@ ClassMethod GetPublicFileOutputTranslateTable(charset As %String) As %Status
   return $get(^Port.Configuration("io.public.out"), "UTF8")
 }
 
+ClassMethod SetTestFormat(type As %String)
+{
+  set type = $$$ucase(type)
+  if '$lf($lb("UDL", "XML"), type) quit
+  set ^Port.Configuration("test.format") = type
+}
+
+ClassMethod GetTestFormat(type As %String)
+{
+  return $get(^Port.Configuration("test.format"), "XML")
+}
+
 ClassMethod RewritePlaceHolders(basePath As %String, projectName As %String = {$get(^||Port.Project)}, allowedPlaceholders = "*", params... As %String) As %String [ Internal ]
 {
 

cls/Port/Installer.cls

@@ -40,12 +40,12 @@ XData LocaleEN [ XMLNamespace = "https://github.com/rfns/port" ]
     <Message Id="-303" Name="ExportingType">Exporting %1 %2 ...</Message>
     <Message Id="-304" Name="NewWorkspaceDetected">A new workspace has been detected for %1. The project will be exported as a whole.</Message>
     <Message Id="-400" Name="SynchronizingProject">Synchronizing the project %1.</Message>
-    <Message Id="-401" Name="CheckingExtraneousItems">Checking for the presence of orphan items in the project.</Message>
-    <Message Id="-402" Name="TotalExtraneousItems">Total orphan items detected: %1.</Message>
-    <Message Id="-403" Name="NoExtraneousItems">No orphan items were found.</Message>
+    <Message Id="-401" Name="CheckingExtraneousItems">Checking the project for extraneous items ...</Message>
+    <Message Id="-402" Name="TotalExtraneousItems">Total extraneous items found: %1.</Message>
+    <Message Id="-403" Name="NoExtraneousItems">No extraneous items were found.</Message>
     <Message Id="-404" Name="RemovingFile">Removing %1 (file).</Message>
-    <Message Id="-405" Name="RemovingDirectory">Removing item %1 (directory).</Message>
-    <Message Id="-406" Name="TotalExtraneousItemsRemoved">Total of removed orphan items: %1</Message>
+    <Message Id="-405" Name="RemovingDirectory">Removing %1 (directory).</Message>
+    <Message Id="-406" Name="TotalExtraneousItemsRemoved">Total of extraneous items removed: %1</Message>
     <Message Id="-500" Name="Done"> done.</Message>
     <Message Id="-501" Name="DoneWithErrors"> done, but with errors.</Message>
     <Message Id="-501" Name="Failed"> failed. Reason: %1.</Message>
@@ -217,12 +217,12 @@ XData LocalePTBR [ XMLNamespace = "https://github.com/rnfs/port" ]
     <Message Id="-303" Name="ExportingType">Exportando %1 %2 ...</Message>
     <Message Id="-304" Name="NewWorkspaceDetected">Um novo espaço de trabalho foi detectado para %1. O projeto será exportado como um todo.</Message>
     <Message Id="-400" Name="SynchronizingProject">Sincronizando o projeto %1.</Message>
-    <Message Id="-401" Name="CheckingExtraneousItems">Verificando presença de itens órfãos no projeto.</Message>
-    <Message Id="-402" Name="TotalExtraneousItems">Total de itens órfãos detectados: %1.</Message>
-    <Message Id="-403" Name="NoExtraneousItems">Nenhum item órfão encontrado.</Message>
-    <Message Id="-404" Name="RemovingFile">Removendo item %1 (arquivo).</Message>
-    <Message Id="-405" Name="RemovingDirectory">Removendo item %1 (diretório).</Message>
-    <Message Id="-406" Name="TotalExtraneousItemsRemoved">Total de itens órfãos removidos: %1.</Message>
+    <Message Id="-401" Name="CheckingExtraneousItems">Verificando o projeto em busca de itens desconhecidos ... </Message>
+    <Message Id="-402" Name="TotalExtraneousItems">Total de items desconhecidos encontrados: %1.</Message>
+    <Message Id="-403" Name="NoExtraneousItems">Não há itens para eliminar.</Message>
+    <Message Id="-404" Name="RemovingFile">Removendo %1 (arquivo).</Message>
+    <Message Id="-405" Name="RemovingDirectory">Removendo %1 (diretório).</Message>
+    <Message Id="-406" Name="TotalExtraneousItemsRemoved">Total de itens desconhecidos removidos: %1.</Message>
     <Message Id="-500" Name="Done"> feito.</Message>
     <Message Id="-501" Name="DoneWithErrors"> feito, mas com erros.</Message>
     <Message Id="-501" Name="Failed"> falhou.</Message>
@@ -375,7 +375,7 @@ ClassMethod Uninstall() As %Status
   return $$$OK
 }
 
-ClassMethod Install(force As %Boolean = 0) As %Status
+ClassMethod Install(force As %Boolean = {$get(^||Port.Envs("FORCE_INSTALL"), 0)}) As %Status
 {
   set sc = $$$OK
   
@@ -424,21 +424,21 @@ ClassMethod Install(force As %Boolean = 0) As %Status
 ClassMethod CreateGlobalMapping() As %Status
 {
   set sc = $$$OK
-  set storage = $get(^Port.Envs("DATABASE"), $namespace)
+  set storage = $get(^||Port.Envs("DATABASE"), $namespace)
   set thisNamespace = $namespace
   new $namespace
   
   set $namespace = "%SYS"
   
-  if ##class(Config.MapGlobals).Exists(storage, "Port.SourceControl*") {
+  if ##class(Config.MapGlobals).Exists(storage, "Port.*") {
     write "> Global is already mapped. Skipping ...", !
     return $$$OK
   }
   
   try {
     $$$ThrowOnError(##class(Config.Namespaces).Get($namespace, .nsProperties))
     set glProperties("Database") = storage
-    $$$ThrowOnError(##class(Config.MapGlobals).Create(thisNamespace, "Port.SourceControl*", .glProperties))    
+    $$$ThrowOnError(##class(Config.MapGlobals).Create(thisNamespace, "Port.*", .glProperties))    
   } catch ex {
     set sc = ex.AsStatus()
   }
@@ -449,7 +449,7 @@ ClassMethod CreateGlobalMapping() As %Status
 ClassMethod RemoveGlobalMapping() As %Status
 {
   set sc = $$$OK
-  set storage = $get(^Port.Envs("DATABASE"), $namespace)
+  set storage = $get(^||Port.Envs("DATABASE"), $namespace)
   set thisNamespace = $namespace    
   new $namespace
   

cls/Port/Project/Base.cls

@@ -55,6 +55,8 @@ Property LogLevel As %Integer [ InitialExpression = 2 ];
 /// A list of files that are ignored by the synchronizer utility.
 Property IgnoredPaths As list Of %String;
 
+Property ProjectName As %String [ Private ];
+
 Method %OnNew(target As %String, logLevel As %Integer = 2) As %Status
 {
   if target = "" {
@@ -63,10 +65,15 @@ Method %OnNew(target As %String, logLevel As %Integer = 2) As %Status
 
   if (target [ "/") || (target [ "\") {
     set workspaceName = $piece(target, $$$PathSlash(target), *)
+
     set project = ##class(%Studio.Project).%OpenId(workspaceName)
+    if '$isobject(project) set project = ##class(%Studio.Project).%New()
+
+    set project.Name = workspaceName
     set fullProjectPath = ##class(%File).NormalizeFilename(target)
     set ..BasePath = ##class(%File).NormalizeFilename(fullProjectPath)
-    set ..Project = project
+    set ..Project = project    
+    set ..ProjectName = workspaceName
     set ..Logger = ##class(Port.Logger).%New(logLevel)
     do ..BuildTypePaths(..BasePath)
     quit $$$OK
@@ -105,7 +112,7 @@ Method LogExclusiveLine(message As %Boolean, logLevel As %Integer = 1, params...
 /// Returns the name of the project that is being used by the utility.
 Method GetProjectName() As %String
 {
-  quit ..Project.Name
+  quit ..ProjectName
 }
 
 /// Adds a path to the IgnoredPaths list. Which is used by the Synchronizer to skip files.

cls/Port/Project/Exporter.cls

@@ -233,13 +233,20 @@ Method ExportClass(className As %String, fromPackage As %Boolean = 0) As %Status
     $$$ThrowOnError(##class(%Compiler.UDL.TextServices).GetTextAsStream($namespace, itemName, .stream))
 
     set file = ##class(%Stream.FileCharacter).%New()
-    set file.TranslateTable = "UTF8"
+    set file.TranslateTable = ##class(Port.Configuration).GetRoutineOutputTranslateTable()
     do file.LinkToFile(destination)
 
     $$$ThrowOnError(file.CopyFromAndSave(stream))
     write ..LogLine($$$Done, 1)
 
     set ..AffectedCount = ..AffectedCount + 1
+    set isUsingXML = (##class(Port.Configuration).GetTestFormat() = "XML")
+
+    if ##class(Port.UnitTest.Util).AssertTestableClass(className, .testableClass) && isUsingXML {
+      set xmlExporter = ##class(Port.Project.XMLExporter).%New(..BasePath, ..LogLevel)
+      do xmlExporter.ExportTest(testableClass, .sc)
+      $$$ThrowOnError(sc)
+    }
   } catch ex {
     set sc = ex.AsStatus()
     write ..LogLine($$$Failed, 1, $System.Status.GetErrorText(sc))

cls/Port/Project/Importer.cls

@@ -49,6 +49,9 @@ Method EnqueueAll() As %Status
 Method EnqueueItem(origin As %String) As %Status
 {
   $$$QuitOnError(..DescribeItem(origin, .isOutdated, .itemName, .itemType, .group))
+
+  // Skip this item because it's invalid.
+  if group = -1 return $$$OK
 
   if '$data(i%ItemsList(itemName)) {
     if '..Backup.IsRequired && '..SkipBackup && ((itemType = "CSP") && isOutdated) {
@@ -118,12 +121,22 @@ Method GetTypePriority(type As %String) As %Integer [ Internal, Private ]
 
 Method DescribeItem(origin As %String, Output isOutdated As %Boolean = 0, Output itemName As %String, Output itemType As %String, Output priority As %String) As %Status [ Internal, Private ]
 {
+
+  set fileName = ##class(%File).GetFilename(origin)     
+  set emptyName = ($piece(fileName, ".") = "")
+  set pathType = ##class(Port.Util).GetPathType(origin)
+
   set isOutdated = 1
   set itemName = ##class(Port.Util).PathToItem(origin)
   set itemType = ##class(Port.Util).GetItemType(itemName)  
-  set priority = ..GetTypePriority(itemType) 
+  set priority = ..GetTypePriority(itemType)
 
-  if '..Overwrite {
+  if emptyName || (pathType '= itemType && (pathType '= "PUBLIC")) {
+    set itemName = ""
+    set itemType = ""
+    set priority = -1
+    set isOutdated = 0
+  } elseif '..Overwrite {
     set isOutdated = ..IsOutdated(itemName, origin, 1)
   }
 
@@ -234,7 +247,7 @@ Method Import() As %Status
       write ..LogLine($$$NoPendingItemsToImport, 0)
     }
 
-    if ..IsNewProject {
+    if ..IsNewProject {      
       write ..LogLine($$$NewProject, 0, ..Project.Name)
     }
 
@@ -255,9 +268,10 @@ Method Import() As %Status
         set ..Backup.IsRequired = 0
       } else  {
         write ..LogLine($$$Failed, 0)
-        $$$ThrowOnError($$$PERROR(UnableToRemoveDirectory, ..BackupDirectory))
+        $$$ThrowOnError($$$PERROR($$$UnableToRemoveDirectory, ..BackupDirectory))
       }
     }
+    $$$ThrowOnError(..Project.%Save())
     tcommit
   } catch ex {
     set sc = ex.AsStatus()
@@ -293,29 +307,44 @@ ClassMethod ImportFromExternalSource(itemName As %String, origin As %String, ite
     $$$QuitOnError(fs.LinkToFile(origin))
     $$$QuitOnError(##class(%Compiler.UDL.TextServices).SetTextFromStream($namespace, itemName, fs))
   } elseif ##class(Port.Util).IsRoutine(itemName) || (itemType = "CSP") {
-    set routine = ""
-    set fs = ##class(%Stream.FileCharacter).%New()
-    set fs.TranslateTable = ##class(Port.Configuration).GetRoutineInputTranslateTable()
+    if itemName [ "/" set itemName = $$$LPadProvidedSlash(itemName, "/")
 
-    if itemType = "CSP" {
-      set extension = $$$ucase($piece(origin, ".", *))
-      if extension = "" set extension = "TXT"
-      do ##class(%CSP.StreamServer).FileClassify(extension, .type, .bin)
-      if bin set fs = ##class(%Stream.FileBinary).%New()
-      else  set fs.TranslateTable = ##class(Port.Configuration).GetPublicFileInputTranslateTable()
-    }
-
-    $$$QuitOnError(fs.LinkToFile(origin))
-
     if ##class(%RoutineMgr).Exists(itemName) {
-      set routine = ##class(%RoutineMgr).%OpenId(itemName)     
+      set routine = ##class(%RoutineMgr).%OpenId(itemName)
     } else {
       set routine = ##class(%RoutineMgr).%New(itemName)
     }
-
+
+    set localSource = ##class(%Stream.FileCharacter).%New()
+    set localSource.TranslateTable = ##class(Port.Configuration).GetRoutineInputTranslateTable()    
+
+    if itemType = "CSP" {  
+      if routine = "" return $$$PERROR($$$NoCompatibleApplicationForPublicFile, origin)
+      set extension = $$$ucase($piece(origin, ".", *))
+      if extension = "" set extension = "TXT"
+      do ##class(%CSP.StreamServer).FileClassify(extension, .type, .bin)
+
+      /// We can't use any translation table when importing a binary, so we do a bit-to-bit copy.
+      if bin {
+        set destination = routine.Code.Filename
+        set path = ##class(%File).GetDirectory(routine.Code.Filename)        
+        set routine = ""
+        set localSource = ""
+        $$$QuitOnError(##class(Port.Util).EnsurePathExists(path))
+        $$$QuitOnError(##class(%File).CopyFile(origin, destination))
+        return $$$OK
+      } else {
+        set tt = ##class(Port.Configuration).GetPublicFileInputTranslateTable()
+        set localSource.TranslateTable = tt
+        set routine.Code.TranslateTable = tt
+      }
+    }    
+
     set code = routine.Code
     do code.Clear()
-    $$$QuitOnError(code.CopyFrom(fs))
+
+    $$$QuitOnError(localSource.LinkToFile(origin))
+    $$$QuitOnError(code.CopyFrom(localSource))
     $$$QuitOnError(routine.%Save())
   }
   quit $$$OK

cls/Port/Project/Manager.cls

@@ -107,10 +107,9 @@ ClassMethod RemoveItemsByType(projectName As %String, type As %String, Output it
 
 ClassMethod ExportTestSuiteToXML(outputPath As %String) As %Status
 {
-  set testPath = ##class(Port.Configuration).GetTestPath()
   set exporter = ..InstantiateTool("X", outputPath, 2, .sc)
   $$$QuitOnError(sc)
-  return exporter.ExportTests(outputPath_"/"_testPath)
+  return exporter.ExportTests()
 }
 
 ClassMethod LogFromImportList(projectName As %String, ByRef importList As %String) [ Final, Internal, Private ]