diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..21f62dd --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea/ +.git/ + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1a939fd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 gkiryaziev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 098c941..e70e51e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ ## WordList Cleaner v.0.2.6 +### Install: +``` +go get github.com/gkiryaziev/go-wordlist-cleaner +``` + Remove non-printable words, trim words length, search duplicates, sorting, words counting. ![Alt text](/screenshot.jpg?raw=true "Usage") @@ -18,7 +23,7 @@ Sequence of the option keys is not critical. 13.10.2015 - Changed algorithm and view for progress bar. Fixed calculation for strings in files. -Examples: +### Examples: ``` wordlistcleaner.exe -min 8 -max 10 -src Source.dic -new New.dic remove trim wordlistcleaner.exe -src Source.dic -new New.dic trim diff --git a/main.go b/main.go index 2a9a532..16badfc 100644 --- a/main.go +++ b/main.go @@ -8,11 +8,11 @@ import ( "strconv" "time" - o "./operations" - s "./service" + o "github.com/gkiryaziev/go-wordlist-cleaner/operations" + s "github.com/gkiryaziev/go-wordlist-cleaner/service" ) -// Do job +// doJob is main function func doJob(remove, trim, duplicate, sorting, calculate bool, min, max int, src_file, new_file string) error { // Check operations diff --git a/operations/calculate.go b/operations/calculate.go index d76aefb..7d643ab 100644 --- a/operations/calculate.go +++ b/operations/calculate.go @@ -3,10 +3,10 @@ package operations import ( "fmt" - s "../service" + s "github.com/gkiryaziev/go-wordlist-cleaner/service" ) -// Calculate lines in source file +// DoCalculate calculate lines in source file func DoCalculate(src_file string) error { total, err := s.CalculateLines(src_file) diff --git a/operations/duplicate_search.go b/operations/duplicate_search.go index 821fd66..1371a7b 100644 --- a/operations/duplicate_search.go +++ b/operations/duplicate_search.go @@ -3,14 +3,15 @@ package operations import ( "bufio" "fmt" - "github.com/cheggaaa/pb" "os" //"time" - s "../service" + "github.com/cheggaaa/pb" + + s "github.com/gkiryaziev/go-wordlist-cleaner/service" ) -// Search duplicates in source file and write uniq to new file +// DoDuplicate search duplicates in source file and write uniq to new file func DoDuplicate(src_file, new_file string) error { m := map[uint64]bool{} diff --git a/operations/remove_trim.go b/operations/remove_trim.go index 07007b6..fed5cf0 100644 --- a/operations/remove_trim.go +++ b/operations/remove_trim.go @@ -7,9 +7,10 @@ import ( "os" //"time" - s "../service" + s "github.com/gkiryaziev/go-wordlist-cleaner/service" ) +// isPrint check printable characters func isPrint(text string) bool { for _, r := range text { if r < 32 || r > 126 { @@ -19,6 +20,7 @@ func isPrint(text string) bool { return true } +// isSize check string size func isSize(min, max int, line string) bool { if len([]rune(line)) < min || len([]rune(line)) > max { return false @@ -26,6 +28,7 @@ func isSize(min, max int, line string) bool { return true } +// DoClean clean a string func DoClean(remove, trim bool, min, max int, src_file, new_file string) error { var added int64 = 0 diff --git a/operations/sorting.go b/operations/sorting.go index bd6fc2d..cde4c93 100644 --- a/operations/sorting.go +++ b/operations/sorting.go @@ -4,10 +4,10 @@ import ( "fmt" "sort" - s "../service" + s "github.com/gkiryaziev/go-wordlist-cleaner/service" ) -// Read source file, sort it alphabetically and write to new file +// DoSorting read source file, sort it alphabetically and write to new file func DoSorting(src_file, new_file string) error { total, err := s.CalculateLines(src_file) diff --git a/service/calculate_hash.go b/service/calculate_hash.go index 06af8b3..dbc9b5c 100644 --- a/service/calculate_hash.go +++ b/service/calculate_hash.go @@ -4,12 +4,14 @@ import ( "hash/fnv" ) +// GetHashFvn return uint32 fnv hash func GetHashFvn(s string) uint32 { h := fnv.New32a() h.Write([]byte(s)) return h.Sum32() } +// GetHashFvn return uint64 fnv hash func GetHashFvn64(s string) uint64 { h := fnv.New64a() h.Write([]byte(s)) diff --git a/service/calculate_lines.go b/service/calculate_lines.go index 1986a2f..79a7d3d 100644 --- a/service/calculate_lines.go +++ b/service/calculate_lines.go @@ -6,6 +6,7 @@ import ( "os" ) +// CalculateLines return count of lines func CalculateLines(path string) (int64, error) { file, err := os.Open(path) if err != nil { diff --git a/service/check_all.go b/service/check_all.go index 2217398..9c4d6a4 100644 --- a/service/check_all.go +++ b/service/check_all.go @@ -6,7 +6,7 @@ import ( "os" ) -// Main check error +// CheckError check error func CheckError(err error) { if err != nil { Usage() @@ -16,7 +16,7 @@ func CheckError(err error) { } } -// Check is file or directory exist +// CheckFile check is file or directory exist func CheckFile(file string) error { f, err := os.Stat(file) if err == nil { @@ -29,7 +29,7 @@ func CheckFile(file string) error { return nil } -// Check key value +// CheckArgs check key value func CheckArgs(args_length, arg_index int) error { if args_length == (arg_index + 1) { return errors.New("Not specified key value.") diff --git a/service/readwrite_file.go b/service/readwrite_file.go index 1d05b6f..0725f88 100644 --- a/service/readwrite_file.go +++ b/service/readwrite_file.go @@ -6,7 +6,7 @@ import ( "os" ) -// Append lines to file +// AppendLine append lines to file func AppendLine(line, path string) error { file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { @@ -20,7 +20,7 @@ func AppendLine(line, path string) error { return nil } -// Read file line by line +// ReadLine read file line by line func ReadLine(path string) ([]string, error) { file, err := os.Open(path) if err != nil { @@ -38,7 +38,7 @@ func ReadLine(path string) ([]string, error) { return lines, scanner.Err() } -// Write file line by line +// WriteLine write file line by line func WriteLine(lines []string, path string) error { file, err := os.Create(path) if err != nil { diff --git a/service/search_files.go b/service/search_files.go index 3302fe4..df69f17 100644 --- a/service/search_files.go +++ b/service/search_files.go @@ -6,7 +6,7 @@ import ( "path/filepath" ) -// Search files in directory by extension +// SearchFilesInDir return list of files in directory by extension func SearchFilesInDir(file_ext, path string) ([]string, error) { var files_list []string