From 0367f2561b6c56ee90ba7276fcdb67f9fca59d32 Mon Sep 17 00:00:00 2001 From: sisong Date: Mon, 29 May 2023 14:10:11 +0800 Subject: [PATCH 1/7] update vc build & libs; --- HDiffPatch | 2 +- builds/vc/ApkNormalized.vcxproj.user | 3 - builds/vc/ZipDiff.vcxproj.user | 3 - builds/vc/ZipPatch.vcxproj.user | 3 - builds/vc2019/ApkDiffPatch.sln | 46 ------ builds/vc2019/ApkNormalized.vcxproj | 176 ----------------------- builds/vc2019/ZipDiff.vcxproj | 203 --------------------------- builds/vc2019/ZipPatch.vcxproj | 181 ------------------------ lzma | 2 +- 9 files changed, 2 insertions(+), 617 deletions(-) delete mode 100644 builds/vc/ApkNormalized.vcxproj.user delete mode 100644 builds/vc/ZipDiff.vcxproj.user delete mode 100644 builds/vc/ZipPatch.vcxproj.user delete mode 100644 builds/vc2019/ApkDiffPatch.sln delete mode 100644 builds/vc2019/ApkNormalized.vcxproj delete mode 100644 builds/vc2019/ZipDiff.vcxproj delete mode 100644 builds/vc2019/ZipPatch.vcxproj diff --git a/HDiffPatch b/HDiffPatch index b9506ba..b2555ed 160000 --- a/HDiffPatch +++ b/HDiffPatch @@ -1 +1 @@ -Subproject commit b9506ba046604a296cbbe20528dede8c9279b64c +Subproject commit b2555ed6bcd9d6d9baef0609f43b5707e771180a diff --git a/builds/vc/ApkNormalized.vcxproj.user b/builds/vc/ApkNormalized.vcxproj.user deleted file mode 100644 index 695b5c7..0000000 --- a/builds/vc/ApkNormalized.vcxproj.user +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/builds/vc/ZipDiff.vcxproj.user b/builds/vc/ZipDiff.vcxproj.user deleted file mode 100644 index 6847b6b..0000000 --- a/builds/vc/ZipDiff.vcxproj.user +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/builds/vc/ZipPatch.vcxproj.user b/builds/vc/ZipPatch.vcxproj.user deleted file mode 100644 index 695b5c7..0000000 --- a/builds/vc/ZipPatch.vcxproj.user +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file diff --git a/builds/vc2019/ApkDiffPatch.sln b/builds/vc2019/ApkDiffPatch.sln deleted file mode 100644 index 69ce74c..0000000 --- a/builds/vc2019/ApkDiffPatch.sln +++ /dev/null @@ -1,46 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2012 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ApkNormalized", "ApkNormalized.vcxproj", "{B7BA5C32-5DC9-431D-BFD7-35ED679CD797}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZipPatch", "ZipPatch.vcxproj", "{69753A53-0320-4AF9-ADE7-A70F0C38358F}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZipDiff", "ZipDiff.vcxproj", "{96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797}.Debug|x64.ActiveCfg = Debug|x64 - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797}.Debug|x64.Build.0 = Debug|x64 - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797}.Debug|x86.ActiveCfg = Debug|Win32 - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797}.Debug|x86.Build.0 = Debug|Win32 - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797}.Release|x64.ActiveCfg = Release|x64 - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797}.Release|x64.Build.0 = Release|x64 - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797}.Release|x86.ActiveCfg = Release|Win32 - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797}.Release|x86.Build.0 = Release|Win32 - {69753A53-0320-4AF9-ADE7-A70F0C38358F}.Debug|x64.ActiveCfg = Debug|x64 - {69753A53-0320-4AF9-ADE7-A70F0C38358F}.Debug|x64.Build.0 = Debug|x64 - {69753A53-0320-4AF9-ADE7-A70F0C38358F}.Debug|x86.ActiveCfg = Debug|Win32 - {69753A53-0320-4AF9-ADE7-A70F0C38358F}.Debug|x86.Build.0 = Debug|Win32 - {69753A53-0320-4AF9-ADE7-A70F0C38358F}.Release|x64.ActiveCfg = Release|x64 - {69753A53-0320-4AF9-ADE7-A70F0C38358F}.Release|x64.Build.0 = Release|x64 - {69753A53-0320-4AF9-ADE7-A70F0C38358F}.Release|x86.ActiveCfg = Release|Win32 - {69753A53-0320-4AF9-ADE7-A70F0C38358F}.Release|x86.Build.0 = Release|Win32 - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}.Debug|x64.ActiveCfg = Debug|x64 - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}.Debug|x64.Build.0 = Debug|x64 - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}.Debug|x86.ActiveCfg = Debug|Win32 - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}.Debug|x86.Build.0 = Debug|Win32 - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}.Release|x64.ActiveCfg = Release|x64 - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}.Release|x64.Build.0 = Release|x64 - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}.Release|x86.ActiveCfg = Release|Win32 - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA}.Release|x86.Build.0 = Release|Win32 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/builds/vc2019/ApkNormalized.vcxproj b/builds/vc2019/ApkNormalized.vcxproj deleted file mode 100644 index 1e6b13c..0000000 --- a/builds/vc2019/ApkNormalized.vcxproj +++ /dev/null @@ -1,176 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - - - - - - - - - - - - - - - - - - {B7BA5C32-5DC9-431D-BFD7-35ED679CD797} - v4.0 - ManagedCProj - ApkNormalized - - - - Application - true - false - Unicode - v142 - - - Application - true - false - Unicode - v142 - - - Application - false - false - Unicode - true - v142 - - - Application - false - false - Unicode - true - v142 - - - - - - - - - - - - - - - - - - - true - $(Configuration)\obj\ - - - true - $(Platform)\$(Configuration)\obj\ - - - false - AllRules.ruleset - $(Configuration)\obj\ - - - false - AllRules.ruleset - $(Platform)\$(Configuration)\obj\ - - - - Level3 - Disabled - WIN32;_DEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) - Default - /wd4819 %(AdditionalOptions) - - - true - - - - - Level3 - Disabled - WIN32;_DEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) - Default - /wd4819 %(AdditionalOptions) - - - true - - - - - Level3 - WIN32;NDEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) - Default - Speed - true - true - MultiThreaded - false - true - Fast - /wd4819 %(AdditionalOptions) - - - false - - - - - Level3 - WIN32;NDEBUG;_IS_USED_MULTITHREAD=0;%(PreprocessorDefinitions) - Default - Speed - true - true - MultiThreaded - false - true - Fast - FastCall - /wd4819 %(AdditionalOptions) - - - false - - - - - - \ No newline at end of file diff --git a/builds/vc2019/ZipDiff.vcxproj b/builds/vc2019/ZipDiff.vcxproj deleted file mode 100644 index 618fd35..0000000 --- a/builds/vc2019/ZipDiff.vcxproj +++ /dev/null @@ -1,203 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {96E107FE-4A2E-4008-9FDE-8CEAA695BFCA} - v4.0 - ManagedCProj - ZipDiff - - - - Application - true - false - Unicode - false - v142 - - - Application - true - false - Unicode - false - v142 - - - Application - false - false - Unicode - true - v142 - - - Application - false - false - Unicode - true - v142 - - - - - - - - - - - - - - - - - - - true - - - true - - - false - AllRules.ruleset - - - false - AllRules.ruleset - - - - Level3 - Disabled - WIN32;_DEBUG;%(PreprocessorDefinitions) - Default - /wd4819 %(AdditionalOptions) - - - true - - - - - Level3 - Disabled - WIN32;_DEBUG;%(PreprocessorDefinitions) - Default - /wd4819 %(AdditionalOptions) - - - true - - - - - Level3 - WIN32;NDEBUG;%(PreprocessorDefinitions) - Default - Speed - true - true - MultiThreaded - false - true - Fast - /wd4819 %(AdditionalOptions) - - - false - - - - - Level3 - WIN32;NDEBUG;%(PreprocessorDefinitions) - Default - Speed - true - true - MultiThreaded - false - true - Fast - FastCall - /wd4819 %(AdditionalOptions) - - - false - - - - - - \ No newline at end of file diff --git a/builds/vc2019/ZipPatch.vcxproj b/builds/vc2019/ZipPatch.vcxproj deleted file mode 100644 index c71e773..0000000 --- a/builds/vc2019/ZipPatch.vcxproj +++ /dev/null @@ -1,181 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {69753A53-0320-4AF9-ADE7-A70F0C38358F} - v4.0 - ManagedCProj - ZipPatch - - - - Application - true - false - Unicode - v142 - - - Application - true - false - Unicode - v142 - - - Application - false - false - Unicode - true - v142 - - - Application - false - false - Unicode - true - v142 - - - - - - - - - - - - - - - - - - - true - $(Configuration)\pobj\ - - - true - $(Platform)\$(Configuration)\pobj\ - - - false - AllRules.ruleset - $(Configuration)\pobj\ - - - false - AllRules.ruleset - $(Platform)\$(Configuration)\pobj\ - - - - Level3 - Disabled - WIN32;_DEBUG;%(PreprocessorDefinitions) - Default - /wd4819 %(AdditionalOptions) - - - true - - - - - Level3 - Disabled - WIN32;_DEBUG;%(PreprocessorDefinitions) - Default - /wd4819 %(AdditionalOptions) - - - true - - - - - Level3 - WIN32;NDEBUG;%(PreprocessorDefinitions) - Default - Speed - true - true - MultiThreaded - false - true - Fast - /wd4819 %(AdditionalOptions) - - - false - - - - - Level3 - WIN32;NDEBUG;%(PreprocessorDefinitions) - Default - Speed - true - true - MultiThreaded - false - true - Fast - FastCall - /wd4819 %(AdditionalOptions) - - - false - - - - - - \ No newline at end of file diff --git a/lzma b/lzma index ec6ab5a..af82ecb 160000 --- a/lzma +++ b/lzma @@ -1 +1 @@ -Subproject commit ec6ab5acf2fb90653cbb94e64c86884b13e5db1f +Subproject commit af82ecb1dc06f65c367debebdda8aad10ba7ad3e From 6e0a1bb1c2441ceb5b4fb5c49c97d4d6f4511356 Mon Sep 17 00:00:00 2001 From: sisong Date: Mon, 29 May 2023 14:11:28 +0800 Subject: [PATCH 2/7] dispose res when openfile fail; --- src/patch/Zipper.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/patch/Zipper.cpp b/src/patch/Zipper.cpp index ab0b06d..b3cfd14 100644 --- a/src/patch/Zipper.cpp +++ b/src/patch/Zipper.cpp @@ -904,8 +904,13 @@ bool Zipper_openFile(Zipper* self,const char* zipFileName,int fileEntryMaxCount, assert(self->_fileStream.m_file==0); check(hpatch_TFileStreamOutput_open(&self->_fileStream,zipFileName,(hpatch_StreamPos_t)(-1))); hpatch_TFileStreamOutput_setRandomOut(&self->_fileStream,hpatch_TRUE); - return Zipper_openStream(self,&self->_fileStream.base,fileEntryMaxCount, - ZipAlignSize,compressLevel,compressMemLevel); + bool result=Zipper_openStream(self,&self->_fileStream.base,fileEntryMaxCount, + ZipAlignSize,compressLevel,compressMemLevel); + if (!result){ + hpatch_TFileStreamOutput_close(&self->_fileStream); + self->_fileStream.m_file=0; + } + return result; } static bool _writeFlush(Zipper* self){ From b3ad509250ae33ccc7a5f17802766f7f36d1f012 Mon Sep 17 00:00:00 2001 From: sisong Date: Thu, 1 Jun 2023 17:39:11 +0800 Subject: [PATCH 3/7] support -SD --- src/diff/DiffData.cpp | 5 +++-- src/diff/DiffData.h | 3 ++- src/diff/Differ.cpp | 47 ++++++++++++++++++++++++++------------- src/diff/Differ.h | 4 ++-- src/patch/Patcher.cpp | 30 ++++++++++++++++++++----- src/patch/Patcher.h | 7 ++++-- src/patch/ZipDiffData.cpp | 2 +- src/zip_diff.cpp | 25 ++++++++++++++++++++- 8 files changed, 94 insertions(+), 29 deletions(-) diff --git a/src/diff/DiffData.cpp b/src/diff/DiffData.cpp index 3cfb8a8..a6c17c6 100644 --- a/src/diff/DiffData.cpp +++ b/src/diff/DiffData.cpp @@ -431,10 +431,11 @@ bool serializeZipDiffData(std::vector& out_data, UnZipper* newZip,UnZippe const std::vector& newRefCompressedSizeList, const std::vector& oldRefList, const std::vector& hdiffzData, - const hdiff_TCompress* compressPlugin){ + const hdiff_TCompress* compressPlugin, + const size_t patchModel){ ZipDiffData data; memset(&data,0,sizeof(ZipDiffData)); - data.PatchModel=0; //now must 0 + data.PatchModel=patchModel; //now must 0 or 1 data.newZipFileCount=UnZipper_fileCount(newZip); data.newZipIsDataNormalized=newZip->_isDataNormalized?1:0; data.newZipAlignSize=newZipAlignSize; diff --git a/src/diff/DiffData.h b/src/diff/DiffData.h index 3323697..1727e67 100644 --- a/src/diff/DiffData.h +++ b/src/diff/DiffData.h @@ -82,6 +82,7 @@ bool serializeZipDiffData(std::vector& out_data, UnZipper* newZip,UnZippe const std::vector& newRefCompressedSizeList, const std::vector& oldRefList, const std::vector& hdiffzData, - const hdiff_TCompress* compressPlugin); + const hdiff_TCompress* compressPlugin, + const size_t patchModel); #endif //ZipDiff_DiffData_h diff --git a/src/diff/Differ.cpp b/src/diff/Differ.cpp index 0384aa8..fdc07e6 100644 --- a/src/diff/Differ.cpp +++ b/src/diff/Differ.cpp @@ -42,9 +42,9 @@ static bool HDiffZ(std::vector& oldData,std::vector& newData,std::vector& out_diffData, - const hdiff_TCompress* compressPlugin,int myBestMatchScore,int threadNum); + const hdiff_TCompress* compressPlugin,int myBestMatchScore,size_t patchStepMemSize,int threadNum); static bool HPatchZ_check(const std::vector& oldData,const std::vector& newData, - const std::vector& diffData,hpatch_TDecompress* decompressPlugin); + const std::vector& diffData,hpatch_TDecompress* decompressPlugin,bool isSD); static bool checkZipInfo(UnZipper* oldZip,UnZipper* newZip); #define check(value) { \ @@ -54,7 +54,7 @@ static bool checkZipInfo(UnZipper* oldZip,UnZipper* newZip); bool ZipDiff(const char* oldZipPath,const char* newZipPath,const char* outDiffFileName, const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin, - int diffMatchScore,bool* out_isNewZipApkV2SignNoError,int threadNum){ + int diffMatchScore,size_t patchStepMemSize,bool* out_isNewZipApkV2SignNoError,int threadNum){ hpatch_TFileStreamInput oldZipStream; hpatch_TFileStreamInput newZipStream; hpatch_TFileStreamOutput outDiffStream; @@ -69,7 +69,7 @@ bool ZipDiff(const char* oldZipPath,const char* newZipPath,const char* outDiffFi check(hpatch_TFileStreamOutput_open(&outDiffStream,outDiffFileName,(hpatch_StreamPos_t)(-1))); hpatch_TFileStreamOutput_setRandomOut(&outDiffStream,hpatch_TRUE); result=ZipDiffWithStream(&oldZipStream.base,&newZipStream.base,&outDiffStream.base, - compressPlugin,decompressPlugin,diffMatchScore, + compressPlugin,decompressPlugin,diffMatchScore,patchStepMemSize, out_isNewZipApkV2SignNoError,threadNum); clear: _isInClear=true; @@ -108,7 +108,8 @@ TCheckZipDiffResult checkZipDiff(const char* oldZipPath,const char* newZipPath,c bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStreamInput* newZipStream, const hpatch_TStreamOutput* outDiffStream, const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin, - int diffMatchScore,bool* out_isNewZipApkV2SignNoError,int threadNum){ + int diffMatchScore,size_t patchStepMemSize, + bool* out_isNewZipApkV2SignNoError,int threadNum){ UnZipper oldZip; UnZipper newZip; std::vector newData; @@ -201,9 +202,10 @@ bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStr } \ check(readZipStreamData(&oldZip,oldRefList,std::vector(),oldData)); _loadNewAndOldData(); - check(HDiffZ(oldData,newData,hdiffzData,compressPlugin,diffMatchScore,threadNum)); + check(HDiffZ(oldData,newData,hdiffzData,compressPlugin,diffMatchScore,patchStepMemSize,threadNum)); _loadNewAndOldData(); - check(HPatchZ_check(oldData,newData,hdiffzData,decompressPlugin)); + const bool isSD=patchStepMemSize>0; + check(HPatchZ_check(oldData,newData,hdiffzData,decompressPlugin,isSD)); { std::vector _empty; oldData.swap(_empty); } { std::vector _empty; newData.swap(_empty); } @@ -211,7 +213,7 @@ bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStr newZipNormalized_compressLevel,newZipNormalized_compressMemLevel, newZip_otherCompressLevel,newZip_otherCompressMemLevel, samePairList,newRefOtherCompressedList,newRefCompressedSizeList, - oldRefList,hdiffzData,compressPlugin)); + oldRefList,hdiffzData,compressPlugin,isSD?1:0)); std::cout<<"ZipDiff size: "<write(outDiffStream,0,out_diffData.data(),out_diffData.data()+out_diffData.size())); @@ -223,7 +225,7 @@ bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStr } static bool HDiffZ(std::vector& oldData,std::vector& newData,std::vector& out_diffData, - const hdiff_TCompress* compressPlugin,int myBestMatchScore,int threadNum){ + const hdiff_TCompress* compressPlugin,int myBestMatchScore,size_t patchStepMemSize,int threadNum){ double time0=clock_s(); const size_t oldDataSize=oldData.size(); const size_t newDataSize=newData.size(); @@ -235,9 +237,16 @@ static bool HDiffZ(std::vector& oldData,std::vector& newData,std:: TByte* oldData0=oldData.data(); const bool isUseBigCacheMatch=true; const size_t matchBlockSize=1024*4; - create_compressed_diff_block(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, - diffData,compressPlugin,myBestMatchScore, - isUseBigCacheMatch,matchBlockSize,threadNum); + if (patchStepMemSize==0){ + create_compressed_diff_block(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, + diffData,compressPlugin,myBestMatchScore, + isUseBigCacheMatch,matchBlockSize,threadNum); + }else{ + std::cout<<" create single compressed diffData...\n"; + create_single_compressed_diff_block(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, + diffData,compressPlugin,myBestMatchScore,patchStepMemSize, + isUseBigCacheMatch,matchBlockSize,threadNum); + } double time1=clock_s(); std::cout<<" diffDataSize: "<& oldData,std::vector& newData,std:: } static bool HPatchZ_check(const std::vector& oldData,const std::vector& newData, - const std::vector& diffData,hpatch_TDecompress* decompressPlugin){ + const std::vector& diffData,hpatch_TDecompress* decompressPlugin,bool isSD){ const size_t oldDataSize=oldData.size(); const size_t newDataSize=newData.size(); const TByte* newData0=newData.data(); const TByte* oldData0=oldData.data(); + double time1=clock_s(); - if (!check_compressed_diff(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, - diffData.data(),diffData.data()+diffData.size(),decompressPlugin)){ + bool checkRet; + if (!isSD){ + checkRet=check_compressed_diff(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, + diffData.data(),diffData.data()+diffData.size(),decompressPlugin); + }else{ + checkRet=check_single_compressed_diff(newData0,newData0+newDataSize,oldData0,oldData0+oldDataSize, + diffData.data(),diffData.data()+diffData.size(),decompressPlugin); + } + if (!checkRet){ std::cout<<"\n hpatchz check hdiffz result error!!!\n"; return false; }else{ diff --git a/src/diff/Differ.h b/src/diff/Differ.h index f1b73e1..3d337b7 100644 --- a/src/diff/Differ.h +++ b/src/diff/Differ.h @@ -42,11 +42,11 @@ typedef enum TCheckZipDiffResult { bool ZipDiff(const char* oldZipPath,const char* newZipPath,const char* outDiffFileName, const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin, - int diffMatchScore,bool* out_isNewZipApkV2SignNoError=0,int threadNum=1); + int diffMatchScore,size_t patchStepMemSize,bool* out_isNewZipApkV2SignNoError=0,int threadNum=1); bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStreamInput* newZipStream, const hpatch_TStreamOutput* outDiffStream, const hdiff_TCompress* compressPlugin,hpatch_TDecompress* decompressPlugin, - int diffMatchScore,bool* out_isNewZipApkV2SignNoError=0,int threadNum=1); + int diffMatchScore,size_t patchStepMemSize,bool* out_isNewZipApkV2SignNoError=0,int threadNum=1); TCheckZipDiffResult checkZipDiff(const char* oldZipPath,const char* newZipPath,const char* diffFileName,int threadNum); TCheckZipDiffResult checkZipDiffWithStream(const hpatch_TStreamInput* oldZipStream, diff --git a/src/patch/Patcher.cpp b/src/patch/Patcher.cpp index e31db99..b43215b 100644 --- a/src/patch/Patcher.cpp +++ b/src/patch/Patcher.cpp @@ -67,8 +67,11 @@ TPatchResult VirtualZipPatchWithStream(const hpatch_TStreamInput* oldZipStream,c bool _isInClear=false; bool isUsedTempFile=false; TByte* temp_cache =0; + size_t temp_cahe_size; ZipFilePos_t decompressSumSize=0; hpatch_TDecompress* decompressPlugin=0; + bool isSD=false; + hpatch_singleCompressedDiffInfo sdiffInfo; UnZipper_init(&oldZip); Zipper_init(&out_newZip); @@ -104,6 +107,7 @@ TPatchResult VirtualZipPatchWithStream(const hpatch_TStreamInput* oldZipStream,c } check(ZipDiffData_openRead(&zipDiffData,zipDiffStream,decompressPlugin),PATCH_ZIPDIFFINFO_ERROR); + isSD=(zipDiffData.PatchModel==1); check(UnZipper_openStream(&oldZip,oldZipStream,zipDiffData.oldZipIsDataNormalized!=0, zipDiffData.oldIsFileDataOffsetMatch!=0),PATCH_OPENREAD_ERROR); check(zipDiffData.oldZipCESize==UnZipper_CESize(&oldZip),PATCH_OLDDATA_ERROR); @@ -113,8 +117,16 @@ TPatchResult VirtualZipPatchWithStream(const hpatch_TStreamInput* oldZipStream,c #endif check(zipDiffData.oldCrc==OldStream_getOldCrc(&oldZip,zipDiffData.oldRefList,zipDiffData.oldRefCount _VIRTUAL_IN(virtual_in)), PATCH_OLDDATA_ERROR); - - check(getCompressedDiffInfo(&diffInfo,zipDiffData.hdiffzData),PATCH_HDIFFINFO_ERROR); + if (!isSD){ + check(getCompressedDiffInfo(&diffInfo,zipDiffData.hdiffzData),PATCH_HDIFFINFO_ERROR); + }else{ + check(getSingleCompressedDiffInfo(&sdiffInfo,zipDiffData.hdiffzData,0),PATCH_SD_HDIFFINFO_ERROR); + memset(&diffInfo,0,sizeof(diffInfo)); + diffInfo.newDataSize=sdiffInfo.newDataSize; + diffInfo.oldDataSize=sdiffInfo.oldDataSize; + diffInfo.compressedCount=(sdiffInfo.compressedSize>0)?1:0; + memcpy(diffInfo.compressType,sdiffInfo.compressType,sizeof(diffInfo.compressType)); + } if (strlen(diffInfo.compressType) > 0) check(decompressPlugin->is_can_open(diffInfo.compressType),PATCH_COMPRESSTYPE_ERROR); @@ -153,10 +165,18 @@ TPatchResult VirtualZipPatchWithStream(const hpatch_TStreamInput* oldZipStream,c zipDiffData.newRefCompressedSizeList,zipDiffData.newRefCompressedSizeCount, threadNum _VIRTUAL_IN(virtual_in) _VIRTUAL_OUT(virtual_out)),PATCH_NEWSTREAM_ERROR); - temp_cache =(TByte*)malloc(HPATCH_CACHE_SIZE); + temp_cahe_size=HPATCH_CACHE_SIZE+(isSD?sdiffInfo.stepMemSize:0); + temp_cache =(TByte*)malloc(temp_cahe_size); check(temp_cache!=0,PATCH_MEM_ERROR); - check(patch_decompress_with_cache(newStream.stream,oldStream.stream,zipDiffData.hdiffzData, - decompressPlugin,temp_cache,temp_cache+HPATCH_CACHE_SIZE),PATCH_HPATCH_ERROR); + if (!isSD){ + check(patch_decompress_with_cache(newStream.stream,oldStream.stream,zipDiffData.hdiffzData, + decompressPlugin,temp_cache,temp_cache+temp_cahe_size),PATCH_HPATCH_ERROR); + }else{ + check(patch_single_compressed_diff(newStream.stream,oldStream.stream,zipDiffData.hdiffzData, + sdiffInfo.diffDataPos,sdiffInfo.uncompressedSize,sdiffInfo.compressedSize, + decompressPlugin,sdiffInfo.coverCount,sdiffInfo.stepMemSize, + temp_cache,temp_cache+temp_cahe_size,0),PATCH_SD_HPATCH_ERROR); + } check(newStream.isFinish,PATCH_ZIPPATCH_ERROR); clear: diff --git a/src/patch/Patcher.h b/src/patch/Patcher.h index 0c6da6a..3d23c50 100644 --- a/src/patch/Patcher.h +++ b/src/patch/Patcher.h @@ -39,15 +39,18 @@ typedef enum TPatchResult { PATCH_OPENWRITE_ERROR, PATCH_CLOSEFILE_ERROR, PATCH_MEM_ERROR, - PATCH_HPATCH_ERROR, + PATCH_HPATCH_ERROR=5, PATCH_HDIFFINFO_ERROR, PATCH_COMPRESSTYPE_ERROR, PATCH_ZIPPATCH_ERROR, PATCH_ZIPDIFFINFO_ERROR, - PATCH_OLDDATA_ERROR, + PATCH_OLDDATA_ERROR=10, PATCH_OLDDECOMPRESS_ERROR, PATCH_OLDSTREAM_ERROR, PATCH_NEWSTREAM_ERROR, + + PATCH_SD_HDIFFINFO_ERROR=20, + PATCH_SD_HPATCH_ERROR, #if (_IS_NEED_VIRTUAL_ZIP) PATCH_VIRTUAL_IN_BEGIN_ERROR=101, diff --git a/src/patch/ZipDiffData.cpp b/src/patch/ZipDiffData.cpp index 007a824..36dfb62 100644 --- a/src/patch/ZipDiffData.cpp +++ b/src/patch/ZipDiffData.cpp @@ -109,7 +109,7 @@ bool ZipDiffData_openRead(ZipDiffData* self,const hpatch_TStreamInput* diffData, //unpack head info const TByte* curBuf=buf; checkUnpackSize(&curBuf,buf+readLen,&self->PatchModel,size_t); - check(self->PatchModel==0);//now must 0 + check(self->PatchModel<=1);//now must 0 or 1 checkUnpackSize(&curBuf,buf+readLen,&self->newZipFileCount,size_t); checkUnpackSize(&curBuf,buf+readLen,&self->newZipIsDataNormalized,size_t); checkUnpackSize(&curBuf,buf+readLen,&self->newZipAlignSize,size_t); diff --git a/src/zip_diff.cpp b/src/zip_diff.cpp index 3b1052a..bf8a050 100644 --- a/src/zip_diff.cpp +++ b/src/zip_diff.cpp @@ -73,6 +73,11 @@ static void printUsage(){ #endif " -m-matchScore\n" " matchScore>=0, DEFAULT -m-3.\n" + " -SD[-stepSize]\n" + " create single compressed diffData(DEFAULT closed), only need one decompress buffer\n" + " when patch, and support step by step patching when step by step downloading!\n" + " stepSize>=" _HDIFFPATCH_EXPAND_AND_QUOTE(hpatch_kStreamCacheSize) ", DEFAULT -SD-2m, recommended 64k,512k,8m etc...\n" + " WARNING: this is new diffFile format, must update patcher for support it.\n" " -d Diff only, do't run patch check, DEFAULT run patch check.\n" " -t Test only, run patch check, ZipPatch(oldZipFile,testDiffFile)==newZipFile ? \n" #if (_IS_USED_MULTITHREAD) @@ -208,6 +213,7 @@ static int _checkSetCompress(hdiff_TCompress** out_compressPlugin, #define _kNULL_VALUE (-1) #define _kNULL_SIZE (~(size_t)0) +#define kDefaultApkPatchStepMemSize (1024*1024*2) #define _THREAD_NUMBER_NULL 0 #define _THREAD_NUMBER_MIN 1 @@ -219,6 +225,8 @@ int zipdiff_cmd_line(int argc, const char * argv[]) { const char* newZipPath =0; const char* outDiffFileName=0; hpatch_BOOL isDiff=_kNULL_VALUE; + hpatch_BOOL isSingleCompressedDiff=_kNULL_VALUE; + size_t patchStepMemSize=0; hpatch_BOOL isPatchCheck = _kNULL_VALUE; hpatch_BOOL isOutputVersion= _kNULL_VALUE; size_t diffMatchScore = _kNULL_SIZE; @@ -267,6 +275,18 @@ int zipdiff_cmd_line(int argc, const char * argv[]) { if (0!=result) return result; } break; + case 'S':{ + _options_check((isSingleCompressedDiff==_kNULL_VALUE) + &&(op[2]=='D')&&((op[3]=='\0')||(op[3]=='-')),"-SD"); + isSingleCompressedDiff=hpatch_TRUE; + if (op[3]=='-'){ + const char* pnum=op+4; + _options_check(kmg_to_size(pnum,strlen(pnum),&patchStepMemSize),"-SD-?"); + _options_check((patchStepMemSize>=hpatch_kStreamCacheSize),"-SD-?"); + }else{ + patchStepMemSize=kDefaultApkPatchStepMemSize; + } + } break; case 'v':{ _options_check((isOutputVersion==_kNULL_VALUE)&&(op[2]=='\0'),"-v"); isOutputVersion=hpatch_TRUE; @@ -307,6 +327,8 @@ int zipdiff_cmd_line(int argc, const char * argv[]) { compressPlugin->setParallelThreadNumber(compressPlugin,(int)threadNum); } + if (isSingleCompressedDiff==_kNULL_VALUE) + isSingleCompressedDiff=hpatch_FALSE; if (diffMatchScore==_kNULL_SIZE) diffMatchScore=kDefaultDiffMatchScore; if (isOutputVersion==_kNULL_VALUE) @@ -340,7 +362,8 @@ int zipdiff_cmd_line(int argc, const char * argv[]) { bool isNewZipApkV2SignNoError=true; if (isDiff){ if (!ZipDiff(oldZipPath,newZipPath,outDiffFileName,compressPlugin,decompressPlugin, - (int)diffMatchScore,&isNewZipApkV2SignNoError,(int)threadNum)){ + (int)diffMatchScore,isSingleCompressedDiff?patchStepMemSize:0, + &isNewZipApkV2SignNoError,(int)threadNum)){ printf("ZipDiff error!\n"); return 1; }//else From 5e60fc0ab17c5ccd6c5d16807e030a62a5abde19 Mon Sep 17 00:00:00 2001 From: sisong Date: Thu, 1 Jun 2023 17:40:58 +0800 Subject: [PATCH 4/7] update version; --- README.md | 2 +- src/patch/patch_types.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32b0f4e..d1d0993 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # [ApkDiffPatch] -[![release](https://img.shields.io/badge/release-v1.5.0-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) +[![release](https://img.shields.io/badge/release-v1.6.0-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/ApkDiffPatch/blob/master/LICENSE) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/ApkDiffPatch/pulls) [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/ApkDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/ApkDiffPatch/issues) diff --git a/src/patch/patch_types.h b/src/patch/patch_types.h index 8e98d67..ff05e24 100644 --- a/src/patch/patch_types.h +++ b/src/patch/patch_types.h @@ -32,7 +32,7 @@ #include //uint32_t uint16_t #define APKDIFFPATCH_VERSION_MAJOR 1 -#define APKDIFFPATCH_VERSION_MINOR 5 +#define APKDIFFPATCH_VERSION_MINOR 6 #define APKDIFFPATCH_VERSION_RELEASE 0 #define _APKDIFFPATCH_VERSION APKDIFFPATCH_VERSION_MAJOR.APKDIFFPATCH_VERSION_MINOR.APKDIFFPATCH_VERSION_RELEASE From c5000e090943625f2a6eb45c6a85c8a0c397369d Mon Sep 17 00:00:00 2001 From: sisong Date: Thu, 1 Jun 2023 17:48:23 +0800 Subject: [PATCH 5/7] fix CI builds; --- .github/workflows/ci.yml | 2 +- .gitignore | 4 ++-- src/diff/Differ.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad05cb..12ecd09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,4 +62,4 @@ jobs: - name: buildByVC run: | git submodule update --init --recursive - msbuild builds/vc2019/ApkDiffPatch.sln -t:rebuild -verbosity:diag -property:Configuration=Release + msbuild builds/vc/ApkDiffPatch.sln -t:rebuild -verbosity:diag -property:Configuration=Release diff --git a/.gitignore b/.gitignore index 2a7774f..78580e9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,5 @@ xcshareddata *.sdf *.ipch xcuserdata/ -builds/vc2019/Release/ -builds/vc2019/x64/ +builds/vc/Release/ +builds/vc/x64/ diff --git a/src/diff/Differ.cpp b/src/diff/Differ.cpp index fdc07e6..949599c 100644 --- a/src/diff/Differ.cpp +++ b/src/diff/Differ.cpp @@ -132,6 +132,7 @@ bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStr int newZip_otherCompressLevel=0; int newZip_otherCompressMemLevel=0; bool newCompressedDataIsNormalized=false; + const bool isSD=patchStepMemSize>0; check(compressPlugin!=0); check(decompressPlugin!=0); @@ -204,7 +205,6 @@ bool ZipDiffWithStream(const hpatch_TStreamInput* oldZipStream,const hpatch_TStr _loadNewAndOldData(); check(HDiffZ(oldData,newData,hdiffzData,compressPlugin,diffMatchScore,patchStepMemSize,threadNum)); _loadNewAndOldData(); - const bool isSD=patchStepMemSize>0; check(HPatchZ_check(oldData,newData,hdiffzData,decompressPlugin,isSD)); { std::vector _empty; oldData.swap(_empty); } { std::vector _empty; newData.swap(_empty); } From e0417cc6f2ce0e04c9d422dfe8cb6bca8ae1048e Mon Sep 17 00:00:00 2001 From: sisong Date: Thu, 1 Jun 2023 18:52:17 +0800 Subject: [PATCH 6/7] fix CI builds; --- .github/workflows/ci.yml | 17 ++++++++++++++++- builds/vc/ApkDiffPatch.sln | 4 +++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12ecd09..fcfaea5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: ndk-build: strategy: matrix: - platform: [ubuntu-latest, macos-latest, windows-latest] + platform: [macos-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v2 @@ -52,6 +52,21 @@ jobs: cd ./builds/android_ndk_jni_mk ndk-build NDK_PROJECT_PATH=. APP_BUILD_SCRIPT=./Android.mk NDK_APPLICATION_MK=./Application.mk + ubuntu-ndk-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: nttld/setup-ndk@v1.0.6 + with: + ndk-version: r16b + - name: buildByAndroidNDK + run: | + sudo apt install libncurses5 + git submodule update --init --recursive + cd ./builds/android_ndk_jni_mk + ndk-build NDK_PROJECT_PATH=. APP_BUILD_SCRIPT=./Android.mk NDK_APPLICATION_MK=./Application.mk + + vc-build: runs-on: windows-latest steps: diff --git a/builds/vc/ApkDiffPatch.sln b/builds/vc/ApkDiffPatch.sln index f147d25..1bd5d3f 100644 --- a/builds/vc/ApkDiffPatch.sln +++ b/builds/vc/ApkDiffPatch.sln @@ -1,6 +1,8 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2012 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.31402.337 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ApkNormalized", "ApkNormalized.vcxproj", "{B7BA5C32-5DC9-431D-BFD7-35ED679CD797}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZipPatch", "ZipPatch.vcxproj", "{69753A53-0320-4AF9-ADE7-A70F0C38358F}" From fa5f954d5fec3fabaac3a99bd40733e3fe310822 Mon Sep 17 00:00:00 2001 From: sisong Date: Thu, 1 Jun 2023 19:01:09 +0800 Subject: [PATCH 7/7] fix CI builds; --- builds/vc/ApkNormalized.vcxproj | 9 +++++---- builds/vc/ZipDiff.vcxproj | 9 +++++---- builds/vc/ZipPatch.vcxproj | 9 +++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/builds/vc/ApkNormalized.vcxproj b/builds/vc/ApkNormalized.vcxproj index 771e855..ec22b09 100644 --- a/builds/vc/ApkNormalized.vcxproj +++ b/builds/vc/ApkNormalized.vcxproj @@ -43,6 +43,7 @@ v4.0 ManagedCProj ApkNormalized + 10.0 @@ -50,14 +51,14 @@ true false Unicode - v110 + v142 Application true false Unicode - v110 + v142 Application @@ -65,7 +66,7 @@ false Unicode true - v110 + v142 Application @@ -73,7 +74,7 @@ false Unicode true - v110 + v142 diff --git a/builds/vc/ZipDiff.vcxproj b/builds/vc/ZipDiff.vcxproj index ca6ae43..a2607f2 100644 --- a/builds/vc/ZipDiff.vcxproj +++ b/builds/vc/ZipDiff.vcxproj @@ -72,6 +72,7 @@ v4.0 ManagedCProj ZipDiff + 10.0 @@ -80,7 +81,7 @@ false Unicode false - v110 + v142 Application @@ -88,7 +89,7 @@ false Unicode false - v110 + v142 Application @@ -96,7 +97,7 @@ false Unicode true - v110 + v142 Application @@ -104,7 +105,7 @@ false Unicode true - v110 + v142 diff --git a/builds/vc/ZipPatch.vcxproj b/builds/vc/ZipPatch.vcxproj index 9baa843..c018218 100644 --- a/builds/vc/ZipPatch.vcxproj +++ b/builds/vc/ZipPatch.vcxproj @@ -48,6 +48,7 @@ v4.0 ManagedCProj ZipPatch + 10.0 @@ -55,14 +56,14 @@ true false Unicode - v110 + v142 Application true false Unicode - v110 + v142 Application @@ -70,7 +71,7 @@ false Unicode true - v110 + v142 Application @@ -78,7 +79,7 @@ false Unicode true - v110 + v142