Skip to content

Commit

Permalink
File extension based charset. (#8)
Browse files Browse the repository at this point in the history 
* adds support for extension based charset

* Update README.md
  • Loading branch information
@rfns
rfns authored Mar 3, 2020
1 parent da4b346 commit db7ef21
Show file tree
Hide file tree
Showing 6 changed files with 143 additions and 84 deletions.
11 changes: 5 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,14 +168,13 @@ For number 3, remember that Port doesn't check the source code but instead their

## I've imported/exported a source code but it seems to have broken my encoding? Strange characters are showing up!

This can be caused due to Port working with UTF-8 by default. You can use the following methods to try and fix it:
This can be caused due to Port working with UTF-8 by default. You can change this behavior according to your preferences by providing exactly which encoding should be used for a specific extension, you can do so by using the following methods:

* If you're having issues with a routine getting __imported__ with wrong encoding, you can use the method `SetRoutineInputTranslateTable`, this includes class items.
* If you're having issues with a routine getting __exported__ with wrong encoding, you can use the method `SetRoutineOutputTranslateTable`, this also includes class items.
* If you're having issues __importing__ with any files are that aren't routines, which means that they're public type classified, including CSP files. Then you can use the method `SetPublicFileInputTranslateTable`.
* Finally, you're having issues __exporting__ any files that aren't routines as well, then you can use the method `SetPublicFileOutputTranslateTable`.
* `SetInputCharset("charset")` will transcode the incoming document content to the provided charset.
* `SetOutputCharset("charset")` will transcode the outgoing document content to the provided charset.

> __NOTE:__ All the methods accept only a single parameter which is a string representing the charset. If you don't want to apply any translation, you can pass "RAW" to all methods.
So, let's say if you are using ISO-8859-1 or Windows 1252 when handling CSPs, you can configure _Port_ to use it instead of UTF-8.
Note that by default Studio uses RAW format (which equals to non-unicode) for editing files. This might be a good use case for manually changing the charset for CSP files.

## It seems I'm not able to export GBL or any binary related files?

Expand Down
60 changes: 18 additions & 42 deletions cls/Port/Configuration.cls
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,61 +138,37 @@ ClassMethod RegisterExtendedHooks(implementer As %String) As %Status
return $$$OK
}

ClassMethod GetExtendedHooksImplementer() As %String
{
return $get(^Port.Configuration("source.hooks.custom"))
}

/// Defines the encoding used to save the routine items, this includes class items. Pass RAW to preserve the content as-is.
ClassMethod SetRoutineInputTranslateTable(charset As %String) As %Status
{
set ^Port.Configuration("io.routine.in") = charset
return $$$OK
}

/// Defines the encoding used to save the exported files originated from routines, this includes class items. Pass RAW to preserve the content as-is.
ClassMethod SetRoutineOutputTranslateTable(charset As %String) As %Status
{
set ^Port.Configuration("io.routine.out") = charset
return $$$OK
}

/// Defines the encoding used to save CSP items. Pass RAW to preserve the content as-is.
ClassMethod SetPublicFileInputTranslateTable(charset As %String) As %Status
{
set ^Port.Configuration("io.public.in") = charset
return $$$OK
}

/// Defines the encoding used to save public files originated from CSP items. Pass RAW to preserve the content as-is.
ClassMethod SetPublicFileOutputTranslateTable(charset As %String) As %Status
/// Get the charset to be used when saving a document that has the provided extension.
/// Defaults to "UTF8".
ClassMethod GetInputCharset(extension As %String) As %String
{
set ^Port.Configuration("io.public.out") = charset
return $$$OK
return $get(^Port.Configuration("source.charset.input", $$$ucase(extension)), "UTF8")
}

/// Retrieves the current configured routine translate table for inputs. Defaults to UTF-8.
ClassMethod GetRoutineInputTranslateTable(charset As %String) As %Status
/// Get the charset to be used when retrieving a document that has the provided extension.
/// Defaults to "UTF8".
ClassMethod GetOutputCharset(extension As %String) As %String
{
return $get(^Port.Configuration("io.routine.in"), "UTF8")
return $get(^Port.Configuration("source.charset.output", $$$ucase(extension)), "UTF8")
}

/// Retrieves the current configured routine translate table for outputs. Defaults to UTF-8.
ClassMethod GetRoutineOutputTranslateTable(charset As %String) As %Status
/// Set the character set to be used when saving a document with the provided extension.
ClassMethod SetInputCharset(extension As %String, charset As %String) As %Status
{
return $get(^Port.Configuration("io.routine.out"), "UTF8")
set ^Port.Configuration("source.charset.input", $$$ucase(extension)) = charset
return $$$OK
}

/// Retrieves the current configured routine translate table for outputs. Defaults to UTF-8.
ClassMethod GetPublicFileInputTranslateTable(charset As %String) As %Status
/// Set the character set to be used when retrieving a document with the provided extension.
ClassMethod SetOutputCharset(extension As %String, charset As %String) As %Status
{
return $get(^Port.Configuration("io.public.in"), "UTF8")
set ^Port.Configuration("source.charset.output", $$$ucase(extension)) = charset
return $$$OK
}

/// Retrieves the current configured public file translate table for outputs. Defaults to UTF-8.
ClassMethod GetPublicFileOutputTranslateTable(charset As %String) As %Status
ClassMethod GetExtendedHooksImplementer() As %String
{
return $get(^Port.Configuration("io.public.out"), "UTF8")
return $get(^Port.Configuration("source.hooks.custom"))
}

/// Defines which format the Port should for running unit test. When "XML" is selected, Port will export test classes as xml.
Expand Down
8 changes: 5 additions & 3 deletions cls/Port/Project/Exporter.cls
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ Method ExportPublicFile(itemName As %String) As %Status
}

set workspace = ##class(Port.Configuration).GetWorkspace(..Project.Name)
set extension = $piece(itemName, ".", *)

try {
set csp = ##class(%RoutineMgr).%New(itemName)
Expand All @@ -129,7 +130,7 @@ Method ExportPublicFile(itemName As %String) As %Status

set io = $$$GETIO

$$$SETIO(##class(Port.Configuration).GetPublicFileOutputTranslateTable())
$$$SETIO(##class(Port.Configuration).GetOutputCharset(extension))
set exported = ##class(%File).CopyFile(csp.Code.Filename, destination, 1)
$$$SETIO(io)

Expand Down Expand Up @@ -162,6 +163,7 @@ Method ExportRoutine(name As %String) As %Status
set workspace = ##class(Port.Configuration).GetWorkspace(..Project.Name)
set destination = ##class(Port.Util).ItemToPath(name, workspace)
set destinationLocation = ##class(%File).GetDirectory(destination)
set extension = $piece(name, ".", *)

$$$ThrowOnError(..Synchronizer.AddToWhitelist(name))

Expand All @@ -174,7 +176,7 @@ Method ExportRoutine(name As %String) As %Status
}

set file = ##class(%Stream.FileCharacter).%New()
set file.TranslateTable = ##class(Port.Configuration).GetRoutineOutputTranslateTable()
set file.TranslateTable = ##class(Port.Configuration).GetOutputCharset(extension)
do file.LinkToFile(destination)

set routine = ##class(%RoutineMgr).%OpenId(name)
Expand Down Expand Up @@ -237,7 +239,7 @@ Method ExportClass(className As %String, fromPackage As %Boolean = 0) As %Status
$$$ThrowOnError(##class(%Compiler.UDL.TextServices).GetTextAsStream($namespace, itemName, .stream))

set file = ##class(%Stream.FileCharacter).%New()
set file.TranslateTable = ##class(Port.Configuration).GetRoutineOutputTranslateTable()
set file.TranslateTable = ##class(Port.Configuration).GetOutputCharset("CLS")
do file.LinkToFile(destination)

$$$ThrowOnError(file.CopyFromAndSave(stream))
Expand Down
20 changes: 11 additions & 9 deletions cls/Port/Project/Importer.cls
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,13 +301,16 @@ Method Import() As %Status

ClassMethod ImportFromExternalSource(itemName As %String, origin As %String, itemType As %String) As %Status [ Final, Internal, Private ]
{
if (itemType = "CLS") {
set extension = $$$ucase($piece(origin, ".", *))
set tt = ##class(Port.Configuration).GetInputCharset(extension)

if (itemType = "CLS") {
set fs = ##class(%Stream.FileCharacter).%New()
set fs.TranslateTable = ##class(Port.Configuration).GetRoutineInputTranslateTable()
set fs.TranslateTable = tt
$$$QuitOnError(fs.LinkToFile(origin))
$$$QuitOnError(##class(%Compiler.UDL.TextServices).SetTextFromStream($namespace, itemName, fs))
} elseif ##class(Port.Util).IsRoutine(itemName) || (itemType = "CSP") {
if itemName [ "/" set itemName = $$$LPadProvidedSlash(itemName, "/")
if itemName [ "/" set itemName = $$$LPadProvidedSlash(itemName, "/")

if ##class(%RoutineMgr).Exists(itemName) {
set routine = ##class(%RoutineMgr).%OpenId(itemName)
Expand All @@ -316,14 +319,14 @@ ClassMethod ImportFromExternalSource(itemName As %String, origin As %String, ite
}

set localSource = ##class(%Stream.FileCharacter).%New()
set localSource.TranslateTable = ##class(Port.Configuration).GetRoutineInputTranslateTable()
set localSource.TranslateTable = tt

if itemType = "CSP" {
if routine = "" return $$$PERROR($$$NoCompatibleApplicationForPublicFile, origin)
set extension = $$$ucase($piece(origin, ".", *))
if routine = "" return $$$PERROR($$$NoCompatibleApplicationForPublicFile, origin)
if extension = "" set extension = "TXT"
do ##class(%CSP.StreamServer).FileClassify(extension, .type, .bin)

if extension [ "CSP" || (extension = "CSR") set bin = 0

/// We can't use any translation table when importing a binary, so we do a bit-to-bit copy.
if bin {
set destination = routine.Code.Filename
Expand All @@ -333,8 +336,7 @@ ClassMethod ImportFromExternalSource(itemName As %String, origin As %String, ite
$$$QuitOnError(##class(Port.Util).EnsurePathExists(path))
$$$QuitOnError(##class(%File).CopyFile(origin, destination))
return $$$OK
} else {
set tt = ##class(Port.Configuration).GetPublicFileInputTranslateTable()
} else {
set localSource.TranslateTable = tt
set routine.Code.TranslateTable = tt
}
Expand Down
64 changes: 52 additions & 12 deletions port-prod.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,42 @@ Defines a class used to handle source control hooks after Port execution.</Descr
]]></Implementation>
</Method>

<Method name="GetInputCharset">
<ClassMethod>1</ClassMethod>
<FormalSpec>extension:%String</FormalSpec>
<ReturnType>%String</ReturnType>
<Implementation><![CDATA[ return $get(^Port.Configuration("source.charset.input", $$$ucase(extension)), "UTF-8")
]]></Implementation>
</Method>

<Method name="GetOutputCharset">
<ClassMethod>1</ClassMethod>
<FormalSpec>extension:%String</FormalSpec>
<ReturnType>%String</ReturnType>
<Implementation><![CDATA[ return $get(^Port.Configuration("source.charset.output", $$$ucase(extension)), "UTF-8")
]]></Implementation>
</Method>

<Method name="SetInputCharset">
<ClassMethod>1</ClassMethod>
<FormalSpec>extension:%String,charset:%String</FormalSpec>
<ReturnType>%Status</ReturnType>
<Implementation><![CDATA[
set ^Port.Configuration("source.charset.output", extension) = charset
return $$$OK
]]></Implementation>
</Method>

<Method name="SetOutputCharset">
<ClassMethod>1</ClassMethod>
<FormalSpec>extension:%String,charset:%String</FormalSpec>
<ReturnType>%Status</ReturnType>
<Implementation><![CDATA[
set ^Port.Configuration("source.charset.input", extension) = charset
return $$$OK
]]></Implementation>
</Method>

<Method name="GetExtendedHooksImplementer">
<ClassMethod>1</ClassMethod>
<ReturnType>%String</ReturnType>
Expand Down Expand Up @@ -1556,6 +1592,7 @@ The CSP application name is mirrored into subfolders inside the public folder.</
}
set workspace = ##class(Port.Configuration).GetWorkspace(..Project.Name)
set extension = $piece(itemName, ".", *)
try {
set csp = ##class(%RoutineMgr).%New(itemName)
Expand All @@ -1574,7 +1611,7 @@ The CSP application name is mirrored into subfolders inside the public folder.</
set io = $$$GETIO
$$$SETIO(##class(Port.Configuration).GetPublicFileOutputTranslateTable())
$$$SETIO(##class(Port.Configuration).GetOutputCharset(extension))
set exported = ##class(%File).CopyFile(csp.Code.Filename, destination, 1)
$$$SETIO(io)
Expand Down Expand Up @@ -1611,6 +1648,7 @@ Export a routine to the workspace, the path inside the workspace is taken from t
set workspace = ##class(Port.Configuration).GetWorkspace(..Project.Name)
set destination = ##class(Port.Util).ItemToPath(name, workspace)
set destinationLocation = ##class(%File).GetDirectory(destination)
set extension = $piece(name, ".", *)
$$$ThrowOnError(..Synchronizer.AddToWhitelist(name))
Expand All @@ -1623,7 +1661,7 @@ Export a routine to the workspace, the path inside the workspace is taken from t
}
set file = ##class(%Stream.FileCharacter).%New()
set file.TranslateTable = ##class(Port.Configuration).GetRoutineOutputTranslateTable()
set file.TranslateTable = ##class(Port.Configuration).GetOutputCharset(extension)
do file.LinkToFile(destination)
set routine = ##class(%RoutineMgr).%OpenId(name)
Expand Down Expand Up @@ -1690,7 +1728,7 @@ Export a class to the workspace's 'cls' folder. Each class package is treated li
$$$ThrowOnError(##class(%Compiler.UDL.TextServices).GetTextAsStream($namespace, itemName, .stream))
set file = ##class(%Stream.FileCharacter).%New()
set file.TranslateTable = ##class(Port.Configuration).GetRoutineOutputTranslateTable()
set file.TranslateTable = ##class(Port.Configuration).GetOutputCharset("CLS")
do file.LinkToFile(destination)
$$$ThrowOnError(file.CopyFromAndSave(stream))
Expand Down Expand Up @@ -2090,13 +2128,16 @@ Export a class to the workspace's 'cls' folder. Each class package is treated li
<Private>1</Private>
<ReturnType>%Status</ReturnType>
<Implementation><![CDATA[
if (itemType = "CLS") {
set extension = $$$ucase($piece(origin, ".", *))
set tt = ##class(Port.Configuration).GetInputCharset(extension)
if (itemType = "CLS") {
set fs = ##class(%Stream.FileCharacter).%New()
set fs.TranslateTable = ##class(Port.Configuration).GetRoutineInputTranslateTable()
set fs.TranslateTable = tt
$$$QuitOnError(fs.LinkToFile(origin))
$$$QuitOnError(##class(%Compiler.UDL.TextServices).SetTextFromStream($namespace, itemName, fs))
} elseif ##class(Port.Util).IsRoutine(itemName) || (itemType = "CSP") {
if itemName [ "/" set itemName = $$$LPadProvidedSlash(itemName, "/")
if itemName [ "/" set itemName = $$$LPadProvidedSlash(itemName, "/")
if ##class(%RoutineMgr).Exists(itemName) {
set routine = ##class(%RoutineMgr).%OpenId(itemName)
Expand All @@ -2105,14 +2146,14 @@ Export a class to the workspace's 'cls' folder. Each class package is treated li
}
set localSource = ##class(%Stream.FileCharacter).%New()
set localSource.TranslateTable = ##class(Port.Configuration).GetRoutineInputTranslateTable()
set localSource.TranslateTable = tt
if itemType = "CSP" {
if routine = "" return $$$PERROR($$$NoCompatibleApplicationForPublicFile, origin)
set extension = $$$ucase($piece(origin, ".", *))
if routine = "" return $$$PERROR($$$NoCompatibleApplicationForPublicFile, origin)
if extension = "" set extension = "TXT"
do ##class(%CSP.StreamServer).FileClassify(extension, .type, .bin)
if extension [ "CSP" || (extension = "CSR") set bin = 0
/// We can't use any translation table when importing a binary, so we do a bit-to-bit copy.
if bin {
set destination = routine.Code.Filename
Expand All @@ -2122,8 +2163,7 @@ Export a class to the workspace's 'cls' folder. Each class package is treated li
$$$QuitOnError(##class(Port.Util).EnsurePathExists(path))
$$$QuitOnError(##class(%File).CopyFile(origin, destination))
return $$$OK
} else {
set tt = ##class(Port.Configuration).GetPublicFileInputTranslateTable()
} else {
set localSource.TranslateTable = tt
set routine.Code.TranslateTable = tt
}
Expand Down
Loading

0 comments on commit db7ef21

Please sign in to comment.