Merge pull request #67 from FlameOfIgnis/dev

Release 0.3.0 - Threading/Performance Update
ignis-sec · May 22, 2021 · d78fbf6 · d78fbf6
2 parents 47e623c + 361ccaa
commit d78fbf6
Show file tree

Hide file tree

Showing 35 changed files with 559 additions and 508 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -221,7 +221,7 @@ jobs:
 
   #Integrity check on pull requests with sonar
   sonar:
-    if: github.event_name == 'pull_request'
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     name: Code Integrity Check with Sonarqube
     runs-on: ubuntu-latest
     steps:

diff --git a/Makefile b/Makefile
@@ -5,49 +5,36 @@
 ##############################################################################################################
 
 # Compiler
-CC            := g++
+CC            	:= g++
 #output directory
-BIN            := bin
+BIN            	:= bin
 #include directory
-INCLUDE        := include
+INCLUDE        	:= include
 #Libraries
-LIB            := lib
-LIBRARIES    :=
+LIB            	:= lib
+LIBRARIES    	:=
 
 ifndef PYTHON_VERSION
 PYTHON_VERSION := $(shell python3 -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)";) #3.8
 endif
 
 PYTHON_VERSION_ :=$(shell python$(PYTHON_VERSION) -c "import sys;t='{v[0]}{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)";) #38
 PYTHON_VERSION3 :=$(shell python$(PYTHON_VERSION) -c "import sys;t='{v[0]}.{v[1]}.{v[2]}'.format(v=list(sys.version_info[:3]));sys.stdout.write(t)";) #3.8.s2
-##############################################################################################################
-#####################################     MarkovPassword project options     #################################
-##############################################################################################################
-
-MP_C_FLAGS  := -Wall -Wextra -g
-MP_EXEC     := Markov
-MP_SRC      := $(shell find ./MarkovPasswords/src/ -name '*.cpp') $(shell find ./MarkovModel/src/ -name '*.cpp')
-
-#build pattern
-$(BIN)/$(MP_EXEC): $(MP_SRC)
-	$(CC) $(MP_C_FLAGS) -I$(INCLUDE) -L$(LIB) $^ -o $@ $(LIBRARIES)    
 
-
-PYTHON_VERSION_ :=$(shell python$(PYTHON_VERSION) -c "import sys;t='{v[0]}{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)";) #38
-PYTHON_VERSION3 :=$(shell python$(PYTHON_VERSION) -c "import sys;t='{v[0]}.{v[1]}.{v[2]}'.format(v=list(sys.version_info[:3]));sys.stdout.write(t)";) #3.8.s2
 ##############################################################################################################
 #####################################     MarkovPassword project options     #################################
 ##############################################################################################################
 
 MP_C_FLAGS  := -Wall -Wextra -g
 MP_EXEC     := Markov
-MP_SRC      := $(shell find ./MarkovPasswords/src/ -name '*.cpp') $(shell find ./MarkovModel/src/ -name '*.cpp')
+MP_SRC      := $(shell find ./MarkovPasswords/src/ -name '*.cpp') 
 MP_INC 		:= 
-MP_LIB		:= -lboost_program_options
+MP_LIB		:= -lboost_program_options -lpthread
+MP_INC		:= $(shell pwd)
+
 #build pattern
 $(BIN)/$(MP_EXEC): $(MP_SRC)
-	$(CC) $(MP_C_FLAGS) -I$(MP_INC) -L$(LIB) $^ -o $@ $(MP_LIB)      
-
+	$(CC) $(MP_C_FLAGS) -I$(MP_INC) -L$(LIB) $^ -o $@ $(MP_LIB) 
 
 ##############################################################################################################
 #####################################       MarkovModel project options      #################################
@@ -77,12 +64,13 @@ $(BIN)/%.cpp.o:%.cpp
 #####################################            Markopy Options             #################################
 ##############################################################################################################
 
-MPY_SRC          := $(shell find MarkovModel/src/ -name '*.cpp') MarkovPasswords/src/markovPasswords.cpp $(shell find Markopy/src/Module/ -name '*.cpp')
+MPY_SRC          := MarkovPasswords/src/markovPasswords.cpp MarkovPasswords/src/threadSharedListHandler.cpp $(shell find Markopy/src/Module/ -name '*.cpp')
+MPY_SRC_DIR		 := Markopy/src/
 MPY_OBJS         := $(MPY_SRC:%=$(BIN)/%.o)
 MPY_DEPS         := $(MPY_OBJS:.o=.d)
-MPY_LDFLAGS      := -shared -lboost_python$(PYTHON_VERSION_) -lpython$(PYTHON_VERSION)
+MPY_LDFLAGS      := -shared -lboost_python$(PYTHON_VERSION_) -lpython$(PYTHON_VERSION) -lpthread
 MPY_C_FLAGS      := $(MPY_INC_FLAGS) -MMD -MP -fPIC -I/usr/include/python$(PYTHON_VERSION)
-MPY_INC_DIRS     := $(shell find $(MPY_SRC_DIR) -type d)
+MPY_INC_DIRS     := $(shell find $(MPY_SRC_DIR) -type d) $(shell pwd)
 MPY_INC_FLAGS    := $(addprefix -I,$(MPY_INC_DIRS))
 MPY_SO           := markopy.so
 
@@ -92,7 +80,7 @@ $(BIN)/$(MPY_SO): $(MPY_OBJS)
 # Build step for C++ source
 $(BIN)/%.cpp.o:%.cpp
 	mkdir -p $(dir $@)
-	$(CC) $(MPY_C_FLAGS) -c $< -o $@
+	$(CC) $(MPY_C_FLAGS) $(MPY_INC_FLAGS) -c $< -o $@
 
 -include $(MPY_DEPS)
 

diff --git a/Markopy/Markopy.vcxproj b/Markopy/Markopy.vcxproj
@@ -84,6 +84,7 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
     <TargetExt>.pyd</TargetExt>
+    <IntDir>$(SolutionDir)$(Platform)\intermediate\Markopy</IntDir>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -140,7 +141,8 @@
       <SDLCheck>true</SDLCheck>
       <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
-      <AdditionalIncludeDirectories>$(pythonLocation)\include;$(PYTHON_PATH)\include;$(BOOST_ROOT);$(SolutionDir)MarkovModel\src\;$(SolutionDir)\MarkovPasswords\src\;</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir);$(SolutionDir)\MarkovPasswords\src\;$(SolutionDir)MarkovModel\src\;$(pythonLocation)\include;$(PYTHON_PATH)\include;$(BOOST_ROOT);$(SolutionDir)MarkovModel\src\;$(SolutionDir)\MarkovPasswords\src\;</AdditionalIncludeDirectories>
+      <LanguageStandard>Default</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Console</SubSystem>
@@ -152,10 +154,8 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="..\MarkovModel\src\edge.cpp" />
-    <ClCompile Include="..\MarkovModel\src\model.cpp" />
-    <ClCompile Include="..\MarkovModel\src\node.cpp" />
     <ClCompile Include="..\MarkovPasswords\src\markovPasswords.cpp" />
+    <ClCompile Include="..\MarkovPasswords\src\threadSharedListHandler.cpp" />
     <ClCompile Include="src\Module\markopy.cpp" />
   </ItemGroup>
   <ItemGroup>

diff --git a/Markopy/Markopy.vcxproj.filters b/Markopy/Markopy.vcxproj.filters
@@ -22,9 +22,6 @@
     <Filter Include="Dependencies">
       <UniqueIdentifier>{9fe04840-08b7-4f78-8ae7-6513c3349fd5}</UniqueIdentifier>
     </Filter>
-    <Filter Include="Dependencies\MarkovModel">
-      <UniqueIdentifier>{e1af0caf-1390-43cd-b7d4-57fe1d3ff9dd}</UniqueIdentifier>
-    </Filter>
     <Filter Include="Dependencies\MarkovPasswords">
       <UniqueIdentifier>{ab5a1a08-ca20-48d3-9c82-4fd5e43bb20a}</UniqueIdentifier>
     </Filter>
@@ -33,18 +30,12 @@
     <ClCompile Include="src\Module\markopy.cpp">
       <Filter>Source Files\Module</Filter>
     </ClCompile>
-    <ClCompile Include="..\MarkovModel\src\edge.cpp">
-      <Filter>Dependencies\MarkovModel</Filter>
-    </ClCompile>
-    <ClCompile Include="..\MarkovModel\src\model.cpp">
-      <Filter>Dependencies\MarkovModel</Filter>
-    </ClCompile>
-    <ClCompile Include="..\MarkovModel\src\node.cpp">
-      <Filter>Dependencies\MarkovModel</Filter>
-    </ClCompile>
     <ClCompile Include="..\MarkovPasswords\src\markovPasswords.cpp">
       <Filter>Dependencies\MarkovPasswords</Filter>
     </ClCompile>
+    <ClCompile Include="..\MarkovPasswords\src\threadSharedListHandler.cpp">
+      <Filter>Dependencies\MarkovPasswords</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="src\CLI\markopy_cli.py">

diff --git a/Markopy/src/CLI/markopy_cli.py b/Markopy/src/CLI/markopy_cli.py
@@ -18,15 +18,16 @@
 {__file__} combine -i untrained.mdl -d dataset.dat -s "\\t" -n 500 -w output.txt -o trained.mdl
     Train and immediately generate 500 lines to output.txt. Export trained model.
 """, formatter_class=argparse.RawTextHelpFormatter)
-parser.add_argument("mode",             help="Operation mode, supported modes: \"generate\", \"train\" and \"combine\".")
-parser.add_argument("input",            help="Input model file. This model will be imported before starting operation.")
-parser.add_argument("-o", "--output",   help="Output model file. This model will be exported when done. Will be ignored for generation mode.")
-parser.add_argument("-d", "--dataset",  help="Dataset file to read input from for training. Will be ignored for generation mode.")
-parser.add_argument("-s", "--seperator",help="Seperator character to use with training data.(character between occurrence and value)")
-parser.add_argument("-w", "--wordlist", help="Wordlist file path to export generation results to. Will be ignored for training mode")
-parser.add_argument("--min", default=6, help="Minimum length that is allowed during generation")
-parser.add_argument("--max", default=12,help="Maximum length that is allowed during generation")
-parser.add_argument("-n", "--count",    help="Number of lines to generate. Ignored in training mode.")
+parser.add_argument("mode",                             help="Operation mode, supported modes: \"generate\", \"train\" and \"combine\".")
+parser.add_argument("input",                            help="Input model file. This model will be imported before starting operation.")
+parser.add_argument("-o", "--output",                   help="Output model file. This model will be exported when done. Will be ignored for generation mode.")
+parser.add_argument("-d", "--dataset",                  help="Dataset file to read input from for training. Will be ignored for generation mode.")
+parser.add_argument("-s", "--seperator",                help="Seperator character to use with training data.(character between occurrence and value)")
+parser.add_argument("-w", "--wordlist",                 help="Wordlist file path to export generation results to. Will be ignored for training mode")
+parser.add_argument("--min", default=6,                 help="Minimum length that is allowed during generation")
+parser.add_argument("--max", default=12,                help="Maximum length that is allowed during generation")
+parser.add_argument("-n", "--count",                    help="Number of lines to generate. Ignored in training mode.")
+parser.add_argument("-t", "--threads",default=10,       help="Number of lines to generate. Ignored in training mode.")
 parser.add_argument("-v", "--verbosity",action="count", help="Output verbosity.")
 args = parser.parse_args() 
 
@@ -74,7 +75,7 @@ def cli_train(model, output_forced=False):
         exit(4)
 
     logging.pprint(f'Starting training.', 3)
-    model.Train(args.dataset,args.seperator)
+    model.Train(args.dataset,args.seperator, int(args.threads))
     logging.pprint(f'Training completed.', 2)
 
     if(args.output):
@@ -91,7 +92,7 @@ def cli_generate(model):
     if(os.path.isfile(args.wordlist)):
         logging.pprint(f"{args.wordlist} exists and will be overwritten.", 1)
 
-    model.Generate(int(args.count), args.wordlist, int(args.min), int(args.max))
+    model.Generate(int(args.count), args.wordlist, int(args.min), int(args.max), int(args.threads))
 
 
 

diff --git a/Markopy/src/Module/markopy.cpp b/Markopy/src/Module/markopy.cpp
@@ -7,9 +7,6 @@
 
 using namespace boost::python;
 
-std::random_device rd;
-std::default_random_engine generator(rd());
-std::uniform_int_distribution<long long unsigned> distribution(0, 0xffffFFFF);
 
 BOOST_PYTHON_MODULE(markopy)
 {

diff --git a/MarkovModel/MarkovModel.vcxproj b/MarkovModel/MarkovModel.vcxproj
@@ -112,7 +112,7 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='DLL Model|x64'">
     <LinkIncremental>false</LinkIncremental>
     <OutDir>$(SolutionDir)$(Platform)\Release\</OutDir>
-    <IntDir>$(Platform)\Release\</IntDir>
+    <IntDir>$(SolutionDir)$(Platform)\intermediate\MarkovModel\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Lib Model|x64'">
     <LinkIncremental>false</LinkIncremental>
@@ -204,7 +204,7 @@
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <EnableUAC>false</EnableUAC>
-      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies);cudart.lib</AdditionalDependencies>
+      <AdditionalDependencies>kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
     <CudaLink>
       <LinkOut>$(IntDir)$(TargetName).device-link.obj</LinkOut>
@@ -221,6 +221,7 @@
       <PrecompiledHeader>NotUsing</PrecompiledHeader>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)MarkovModel\src</AdditionalIncludeDirectories>
+      <LanguageStandard>Default</LanguageStandard>
     </ClCompile>
     <Link>
       <SubSystem>Windows</SubSystem>
@@ -243,11 +244,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="src\dllmain.cpp" />
-    <ClCompile Include="src\MarkovModel.h" />
-    <ClCompile Include="src\model.cpp" />
-    <ClCompile Include="src\node.cpp" />
     <ClCompile Include="src\pch.cpp" />
-    <ClCompile Include="src\edge.cpp" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets" />

diff --git a/MarkovModel/MarkovModel.vcxproj.filters b/MarkovModel/MarkovModel.vcxproj.filters
@@ -38,17 +38,5 @@
     <ClCompile Include="src\pch.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="src\node.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="src\model.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="src\edge.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="src\MarkovModel.h">
-      <Filter>Header Files</Filter>
-    </ClCompile>
   </ItemGroup>
 </Project>
diff --git a/MarkovModel/src/MarkovModel.h b/MarkovModel/src/MarkovModel.h
diff --git a/MarkovModel/src/dllmain.cpp b/MarkovModel/src/dllmain.cpp
@@ -2,10 +2,6 @@
 #include "model.h"
 #include <iostream>
 
-// dllmain.cpp : Defines the entry point for the DLL application.
-//std::random_device rd;
-//std::default_random_engine generator(rd());
-//std::uniform_int_distribution<long long unsigned> distribution(0, 0xffffFFFF);
 
 #ifdef _WIN32
 __declspec(dllexport) void dll_loadtest() {

diff --git a/MarkovModel/src/edge.cpp b/MarkovModel/src/edge.cpp