From 9757dcebf63e911c7762817b6a50d50015217c80 Mon Sep 17 00:00:00 2001 From: Ben Boyter Date: Mon, 6 Aug 2018 21:40:34 +1000 Subject: [PATCH] Release 1.6.0 --- README.md | 17 ++++++++++++++--- main.go | 8 +++++++- processor/file.go | 15 +++++++++++++++ processor/processor.go | 4 ++++ 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 78c0e0dae..e283b51d5 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ USAGE: scc DIRECTORY VERSION: - 1.5.0 + 1.6.0 COMMANDS: help, h Shows a list of commands or help for one command @@ -66,7 +66,8 @@ GLOBAL OPTIONS: --complexity, -c Set to skip complexity calculations note will be overridden if wide is set --wide, -w Set to check produce more output such as complexity and code vs complexity ranking. Same as setting format to wide --averagewage value, --aw value Set as integer to set the average wage used for basic COCOMO calculation (default: 56286) - --cocomo, --co Set to check remove cocomo calculation output + --cocomo, --co Set to check remove COCOMO calculation output + --filegccount value, --fgc value How many files to parse before turning the GC on (default: 10000) --debug Set to enable debug output --trace Set to enable trace output, not recommended for multiple files --help, -h show help @@ -107,7 +108,11 @@ Estimated People Required 19.867141 ### Performance -Generally `scc` will be very close to the runtime of `tokei` or faster than any other code counter out there. However if you want greater performance and you have RAM to spare you can disable the garbage collector like the following on linux `GOGC=-1 scc .` which should speed things up considerably. See the below for example runtimes on a 16 CPU Linux machine running against the linux kernel source. +Generally `scc` will be very close to the runtime of `tokei` or faster than any other code counter out there. It is designed to scale to as many CPU's cores as you can provide. + +However if you want greater performance and you have RAM to spare you can disable the garbage collector like the following on linux `GOGC=-1 scc .` which should speed things up considerably. + +See the below for example runtimes on a 16 CPU Linux machine running against the linux kernel source. ``` scc 1.489 s ± 0.055 s @@ -136,6 +141,12 @@ Its possible that you may see the counts vary between runs. This usually means o To help identify this issue run scc like so `scc -v .` and look for the message `too many open files` in the output. If it is there you can rectify it by setting your ulimit to a higher value. +### Low Memory + +If you are running `scc` in a low memory envrionment < 512 MB of RAM you may need to set `--filegccount` or `--fgc` to a lower value such as `0` to force the garbage collector to be on at all times. + +A sign that this is required will be `scc` crashing with panic errors. + ### Tests scc is pretty well tested with many unit, integration and benchmarks to ensure that it is fast and complete. diff --git a/main.go b/main.go index d74235ca2..d3230799a 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,7 @@ func main() { app := cli.NewApp() app.EnableBashCompletion = true app.Name = "scc" - app.Version = "1.5.0" + app.Version = "1.6.0" app.Usage = "Sloc, Cloc and Code. Count lines of code in a directory with complexity estimation." app.UsageText = "scc DIRECTORY" @@ -92,6 +92,12 @@ func main() { Usage: "Set to check remove COCOMO calculation output", Destination: &processor.Cocomo, }, + cli.IntFlag{ + Name: "filegccount, fgc", + Usage: "How many files to parse before turning the GC on", + Destination: &processor.GcFileCount, + Value: 10000, + }, cli.BoolFlag{ Name: "debug", Usage: "Set to enable debug output", diff --git a/processor/file.go b/processor/file.go index 8691efd1e..f2155eed7 100644 --- a/processor/file.go +++ b/processor/file.go @@ -6,6 +6,7 @@ import ( "github.com/monochromegane/go-gitignore" "io/ioutil" "path/filepath" + "runtime/debug" "strings" "sync" ) @@ -67,6 +68,9 @@ func walkDirectoryParallel(root string, output *chan *FileJob) { extensionLookup = wlExtensionLookup } + var mutex = &sync.Mutex{} + totalCount := 0 + var wg sync.WaitGroup all, _ := ioutil.ReadDir(root) // TODO the gitignore should check for futher gitignores deeper in the tree @@ -127,7 +131,15 @@ func walkDirectoryParallel(root string, output *chan *FileJob) { } if ok { + mutex.Lock() + totalCount++ + mutex.Unlock() *output <- &FileJob{Location: root, Filename: info.Name(), Extension: extension, Language: language} + + // Turn GC back to what it was before if we have parsed enough files + if totalCount >= GcFileCount { + debug.SetGCPercent(gcPercent) + } } else if Verbose { printWarn(fmt.Sprintf("skipping file unknown extension: %s", info.Name())) } @@ -152,6 +164,9 @@ func walkDirectoryParallel(root string, output *chan *FileJob) { language, ok := extensionLookup[extension] if ok { + mutex.Lock() + totalCount++ + mutex.Unlock() *output <- &FileJob{Location: filepath.Join(root, f.Name()), Filename: f.Name(), Extension: extension, Language: language} } else if Verbose { printWarn(fmt.Sprintf("skipping file unknown extension: %s", f.Name())) diff --git a/processor/processor.go b/processor/processor.go index 59eed1df3..a5b352431 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "runtime" + "runtime/debug" "sort" "strings" ) @@ -31,6 +32,8 @@ var FileProcessJobQueueSize = runtime.NumCPU() var FileSummaryJobQueueSize = runtime.NumCPU() var WhiteListExtensions = "" var AverageWage int64 = 56286 +var GcFileCount = 10000 +var gcPercent = -1 // Not set via flags but by arguments following the the flags var DirFilePaths = []string{} @@ -43,6 +46,7 @@ var LanguageFeatures = map[string]LanguageFeature{} // Needs to be called at least once in order for anything to actually happen func ProcessConstants() { var database = loadDatabase() + gcPercent = debug.SetGCPercent(gcPercent) startTime := makeTimestampNano() for name, value := range database {