Merge pull request #93 from sisong/dev

fix ApkNormalized error & recode

README.md

@@ -1,5 +1,5 @@
 # [ApkDiffPatch]
-[![release](https://img.shields.io/badge/release-v1.7.0-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) 
+[![release](https://img.shields.io/badge/release-v1.7.1-blue.svg)](https://github.com/sisong/ApkDiffPatch/releases) 
 [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/sisong/ApkDiffPatch/blob/master/LICENSE) 
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-blue.svg)](https://github.com/sisong/ApkDiffPatch/pulls) 
 [![+issue Welcome](https://img.shields.io/github/issues-raw/sisong/ApkDiffPatch?color=green&label=%2Bissue%20welcome)](https://github.com/sisong/ApkDiffPatch/issues)   

src/ZiPatExtraDemo.cpp

@@ -73,9 +73,9 @@ int extra_cmd_line(int argc, const char * argv[]) {
     srcZiPatPath =argv[1];
     outZiPatPath =argv[2];
     const char* appendData=argv[3];//NOTE: modify this for your require
-    printf(" src ZiPat :\"%s\"\n",srcZiPatPath);
-    printf(" out ZiPat :\"%s\"\n",outZiPatPath);
-    printf("test append:\"%s\"\n",appendData);
+    hpatch_printPath_utf8((std::string("src ZiPat  : \"")+srcZiPatPath+"\"\n").c_str());
+    hpatch_printPath_utf8((std::string("out ZiPat  : \"")+outZiPatPath+"\"\n").c_str());
+    hpatch_printPath_utf8((std::string("test append: \"")+appendData+"\"\n").c_str());
     if (!(addToExtra(srcZiPatPath,outZiPatPath,(const TByte*)appendData,strlen(appendData)))){
         return _kRET_ERROR;
     }

src/apk_normalized.cpp

@@ -51,7 +51,7 @@ static void printUsage(){
            "      align file data offset in zip file (compatible with AndroidSDK#zipalign),\n"
            "      remove all data descriptor, reserve & normalized Extra field and Comment,\n"
            "      compatible with jar sign(apk v1 sign), etc...\n"
-           "    if apk file used apk v2 sign, must re sign apk file after ApkNormalized;\n"
+           "    if apk file used apk v2 sign, must re-sign apk file after ApkNormalized;\n"
            "      release signedApk:=AndroidSDK#apksigner(normalizedApk)\n"
            "  -cl-compressLevel\n"
            "    set zlib compress level [0..9], recommended 4,5,6, DEFAULT -cl-6;\n"
@@ -73,6 +73,7 @@ static void printUsage(){
            "                    WARNING: if have compressed empty file,\n"
            "                            it can't patch by old(version<v1.3.5) ZipPatch!\n"
            "      -nce-1        DEFAULT, not compress all empty file.\n"
+           "  -q  quiet mode, don't print fileName\n"
            "  -v  output Version info. \n"
            );
 }
@@ -128,6 +129,10 @@ int normalized_cmd_line(int argc, const char * argv[]){
                 _options_check((isOutputVersion==_kNULL_VALUE)&&(op[2]=='\0'),"-v");
                 isOutputVersion=hpatch_TRUE;
             } break;
+            case 'q':{
+                _options_check((op[2]=='\0'),"-q");
+                g_isPrintApkNormalizedFileName=false;
+            } break;
             case 'n':{
                 if ((op[2]=='c')&&(op[3]=='e')&&(op[4]=='-')&&((op[5]=='0')||(op[5]=='1'))){
                     _options_check(isNotCompressEmptyFile==_kNULL_VALUE,"-nce-?");
@@ -187,7 +192,8 @@ int normalized_cmd_line(int argc, const char * argv[]){
     _options_check(arg_values_size==2,"count");
     const char* srcApk=arg_values[0];
     const char* dstApk=arg_values[1];
-    printf("src: \"%s\"\nout: \"%s\"\n",srcApk,dstApk);
+    hpatch_printPath_utf8((std::string("src: \"")+srcApk+"\"\n").c_str());
+    hpatch_printPath_utf8((std::string("out: \"")+dstApk+"\"\n").c_str());
     double time0=clock_s();
     int apkFilesRemoved=0;
     if (!ZipNormalized(srcApk,dstApk,(int)alignSize,(int)compressLevel,

src/diff/DiffData.cpp

@@ -270,8 +270,8 @@ bool getSamePairList(UnZipper* newZip,UnZipper* oldZip,
                     if (zipFile_name(newZip,i)==zipFile_name(oldZip,oldIndex)) break;
                 }else{
                     printf("WARNING: crc32 equal but data not equal! file index: %d,%d\n",i,oldIndex);
-                    printf("   name:\"%s\"\n        \"%s\"\n",zipFile_name(newZip,i).c_str(),
-                           zipFile_name(oldZip,oldIndex).c_str());
+                    hpatch_printPath_utf8(("   name: \""+zipFile_name(newZip,i)+"\"\n").c_str());
+                    hpatch_printPath_utf8(("         \""+zipFile_name(oldZip,oldIndex)+"\"\n").c_str());
                 }
             }
             if (findSame){

src/diff/Differ.cpp

@@ -308,7 +308,7 @@ static bool checkZipInfo(UnZipper* oldZip,UnZipper* newZip){
     }
     if ((!newIsV2Sign)&&UnZipper_isHaveApkV2orV3SignTag_in_ApkV1SignFile(newZip)){
         //maybe bring apk can't install ERROR!
-        printf("  ERROR: newZip fond \"X-Android-APK-Signed: 2(or 3...)\" in ApkV1Sign file, need re sign "
+        printf("  ERROR: newZip fond \"X-Android-APK-Signed: 2(or 3...)\" in ApkV1Sign file, need re-sign "
                "newZip:=AndroidSDK#apksigner(newZip) before running ZipDiff!\n");
         isOk=false;
     }

src/normalized/normalized.cpp

@@ -31,6 +31,13 @@
 #include <algorithm> //sort
 #include "../patch/Zipper.h"
 #include "../diff/DiffData.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+bool g_isPrintApkNormalizedFileName=true;
+#ifdef __cplusplus
+}
+#endif
 
 #define  check(value) { \
     if (!(value)){ printf(#value" ERROR!\n");  \
@@ -150,7 +157,7 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co
         getAllFiles(&unzipper,files);
         apkFilesRemoved=removeNonEmptyDirs(&unzipper,files);
         fileCount=(int)files.size();
-        std::sort(files.begin(),files.end(),TFileValue::TCmp(fileCount));
+        std::sort(files.begin(),files.end(),TFileValue::TCmp(UnZipper_fileCount(&unzipper)));
         for (int i=0; i<fileCount; ++i) {
             int fileIndex=files[i].fileIndex;
             if (UnZipper_file_isApkV1Sign(&unzipper,fileIndex)){
@@ -177,13 +184,15 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co
     for (int i=0; i<(int)fileIndexs.size(); ++i) {
         int fileIndex=fileIndexs[i];
         std::string fileName=zipFile_name(&unzipper,fileIndex);
-        printf("\"%s\"\n",fileName.c_str());
+        if (g_isPrintApkNormalizedFileName)
+            hpatch_printPath_utf8(("\""+fileName+"\"\n").c_str());
         if (compressLevel==0){
             check(Zipper_file_append_set_new_isCompress(&zipper,false));
         } else if (isCompressedEmptyFile(&unzipper,fileIndex)){
             if (isNotCompressEmptyFile){
                 check(Zipper_file_append_set_new_isCompress(&zipper,false));
-                printf("NOTE: \"%s\" is a compressed empty file, change to uncompressed!\n",fileName.c_str());
+                hpatch_printPath_utf8(("NOTE: \""+fileName+"\"").c_str());
+                printf(" is a compressed empty file, change to uncompressed!\n");
             }else{
                 _compressedEmptyFiles.push_back(fileName);
              }
@@ -206,21 +215,22 @@ bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int co
     check(Zipper_endCentralDirectory_append(&zipper,&unzipper));
 
     for (int i=0;i<(int)_compressedEmptyFiles.size();++i){
-        printf("WARNING: \"%s\" is a compressed empty file, can't patch by old(version<v1.3.5) ZipPatch!)\n",_compressedEmptyFiles[i].c_str());
+        hpatch_printPath_utf8(("WARNING: \""+_compressedEmptyFiles[i]).c_str());
+        printf(" is a compressed empty file, can't patch by old(version<v1.3.5) ZipPatch!)\n");
     }
     if (jarSignFileCount>0){
         if (isHaveApkV2Sign){
-            printf("WARNING: src removed JarSign(ApkV1Sign) (%d file, need re sign)\n",jarSignFileCount);
+            printf("WARNING: src removed JarSign(ApkV1Sign) (%d file, need re-sign)\n",jarSignFileCount);
         }else{
-            printf("NOTE: src found JarSign(ApkV1Sign) (%d file)\n",jarSignFileCount);
+            printf("WARNING: src JarSign(ApkV1Sign) (%d file) has been Normalized, Don't re-sign!\n",jarSignFileCount);
         }
     }
     for (size_t i=0;i<removedFiles.size();++i)
-        printf("WARNING:   removed file: %s\n",removedFiles[i].c_str());
+        hpatch_printPath_utf8((std::string("WARNING:   removed file: \"")+removedFiles[i]+"\"\n").c_str());
     if (isHaveApkV2Sign){
         printf(isHaveApkV3Sign?
-                "WARNING: src removed ApkV2Sign & ApkV3Sign  data (%d Byte, need re sign)\n"
-               :"WARNING: src removed ApkV2Sign data (%d Byte, need re sign)\n",
+                "WARNING: src removed ApkV2Sign & ApkV3Sign  data (%d Byte, need re-sign)\n"
+               :"WARNING: src removed ApkV2Sign data (%d Byte, need re-sign)\n",
                 (int)UnZipper_ApkV2SignSize(&unzipper));
     }
     if (zipper._normalizeSoPageAlignCount>0)

src/normalized/normalized.h

@@ -31,5 +31,11 @@
 //规范化zip包;
 bool ZipNormalized(const char* srcApk,const char* dstApk,int ZipAlignSize,int compressLevel,
                    bool isNotCompressEmptyFile=true,bool isPageAlignSoFile=true,int* out_apkFilesRemoved=0);
-
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern bool g_isPrintApkNormalizedFileName; //default true
+#ifdef __cplusplus
+}
+#endif
 #endif //ZipNormalized_normalized_h

src/patch/NewStream.cpp

@@ -135,63 +135,66 @@ static hpatch_BOOL _NewStream_write(const hpatch_TStreamOutput* stream,
                 check(self->_vout->endVirtual(self->_vout));
 #endif
             ++self->_curFileIndex;
-        }else{
-            break;
-        }
-    }
-
-    if (self->_curFileIndex<self->_fileCount){//open file for write
-        ZipFilePos_t uncompressedSize=UnZipper_file_uncompressedSize(&self->_newZipVCE,self->_curFileIndex);
-        ZipFilePos_t compressedSize=uncompressedSize;
-        if (UnZipper_file_isCompressed(&self->_newZipVCE,self->_curFileIndex)){
-            check(self->_curNewReCompressSizeIndex<self->_newReCompressSizeCount);
-            compressedSize=self->_newReCompressSizeList[self->_curNewReCompressSizeIndex];
-            ++self->_curNewReCompressSizeIndex;
-        }
-        _update_compressedSize(self,self->_curFileIndex,compressedSize);
-
-        bool isWriteOtherCompressedData=(self->_curNewOtherCompressIndex<self->_newRefOtherCompressedCount)
-                &&((int)self->_newRefOtherCompressedList[self->_curNewOtherCompressIndex]==self->_curFileIndex);
-        if (isWriteOtherCompressedData)
-            ++self->_curNewOtherCompressIndex;
+        }else{//open file for write
+            ZipFilePos_t uncompressedSize=UnZipper_file_uncompressedSize(&self->_newZipVCE,self->_curFileIndex);
+            ZipFilePos_t compressedSize=uncompressedSize;
+            if (UnZipper_file_isCompressed(&self->_newZipVCE,self->_curFileIndex)){
+                check(self->_curNewReCompressSizeIndex<self->_newReCompressSizeCount);
+                compressedSize=self->_newReCompressSizeList[self->_curNewReCompressSizeIndex];
+                ++self->_curNewReCompressSizeIndex;
+            }
+            _update_compressedSize(self,self->_curFileIndex,compressedSize);
+
+            bool isWriteOtherCompressedData=(self->_curNewOtherCompressIndex<self->_newRefOtherCompressedCount)
+                    &&((int)self->_newRefOtherCompressedList[self->_curNewOtherCompressIndex]==self->_curFileIndex);
+            if (isWriteOtherCompressedData)
+                ++self->_curNewOtherCompressIndex;
 
-        bool is0FileSize=false; //in zip
-#if (_IS_NEED_VIRTUAL_ZIP)
-        TVirtualZip_out_type ty=kVirtualZip_out_void;
-        if (self->_vout){
-            ty=self->_vout->beginVirtual(self->_vout,&self->_newZipVCE,self->_curFileIndex);
-            switch (ty) {
-                case kVirtualZip_out_void: { //ok, do nothing
-                    assert(self->_vout->virtualStream==0); } break;
-                case kVirtualZip_out_emptyFile_cast: { // set fileSize==0
-                    assert(self->_vout->virtualStream==0);
-                    _update_fileSize(self,self->_curFileIndex,0,0);
-                    is0FileSize=true; } break;
-                case kVirtualZip_out_emptyFile_uncompressed: { //set fileSize==0, need out and endVirtual
-                    assert(self->_vout->virtualStream!=0);
-                    _update_fileSize(self,self->_curFileIndex,0,0);
-                    is0FileSize=true; } break;
-                default: { check(false); } break; //error, or unknow as error
+            bool isVirtualFile0Size=false; // is notInZip virtualFile && uncompressedFizeSize==0 ?
+    #if (_IS_NEED_VIRTUAL_ZIP)
+            TVirtualZip_out_type ty=kVirtualZip_out_void;
+            if (self->_vout){
+                ty=self->_vout->beginVirtual(self->_vout,&self->_newZipVCE,self->_curFileIndex);
+                switch (ty) {
+                    case kVirtualZip_out_void: { //ok, do nothing
+                        assert(self->_vout->virtualStream==0); } break;
+                    case kVirtualZip_out_emptyFile_cast: { // set fileSize==0
+                        assert(self->_vout->virtualStream==0);
+                        _update_fileSize(self,self->_curFileIndex,0,0);
+                        isVirtualFile0Size=true; } break;
+                    case kVirtualZip_out_emptyFile_uncompressed: { //set fileSize==0, need out and endVirtual
+                        assert(self->_vout->virtualStream!=0);
+                        _update_fileSize(self,self->_curFileIndex,0,0);
+                        isVirtualFile0Size=true; } break;
+                    default: { check(false); } break; //error, or unknow as error
+                }
             }
-        }
-#endif
-        if (isWriteOtherCompressedData && self->_newOtherCompressIsValid){
-            check(Zipper_file_append_beginWith(self->_out_newZip,&self->_newZipVCE,self->_curFileIndex,
-                                               false,is0FileSize?0:uncompressedSize,is0FileSize?0:compressedSize,
-                                               self->_newOtherCompressLevel,self->_newOtherCompressMemLevel));
-            self->_curWriteToPosEnd+=uncompressedSize;
-        }else{
-#if (_IS_NEED_VIRTUAL_ZIP)
-            if (ty==kVirtualZip_out_emptyFile_uncompressed){
-                check(!isWriteOtherCompressedData); //now unsupport auto decompress data
+    #endif
+            if (isWriteOtherCompressedData && self->_newOtherCompressIsValid){
+                check(Zipper_file_append_beginWith(self->_out_newZip,&self->_newZipVCE,self->_curFileIndex,
+                                                   false,isVirtualFile0Size?0:uncompressedSize,isVirtualFile0Size?0:compressedSize,
+                                                   self->_newOtherCompressLevel,self->_newOtherCompressMemLevel));
+                self->_curWriteToPosEnd+=uncompressedSize;
+            }else{
+    #if (_IS_NEED_VIRTUAL_ZIP)
+                if (ty==kVirtualZip_out_emptyFile_uncompressed){
+                    check(!isWriteOtherCompressedData); //now unsupport auto decompress data
+                }
+    #endif
+                check(Zipper_file_append_begin(self->_out_newZip,&self->_newZipVCE,self->_curFileIndex,
+                                            isWriteOtherCompressedData,
+                                            isVirtualFile0Size?0:uncompressedSize,isVirtualFile0Size?0:compressedSize));
+                self->_curWriteToPosEnd+=isWriteOtherCompressedData?compressedSize:uncompressedSize;
+            }
+
+            if ((!isVirtualFile0Size)&&(uncompressedSize==0)){ // compressed && empty file 
+                check(Zipper_file_append_part(self->_out_newZip,0,0));
+                check(Zipper_file_append_end(self->_out_newZip));
+                ++self->_curFileIndex;
+            }else{
+                return hpatch_TRUE; //opened a file wait for write;
             }
-#endif
-            check(Zipper_file_append_begin(self->_out_newZip,&self->_newZipVCE,self->_curFileIndex,
-                                           isWriteOtherCompressedData,
-                                           is0FileSize?0:uncompressedSize,is0FileSize?0:compressedSize));
-            self->_curWriteToPosEnd+=isWriteOtherCompressedData?compressedSize:uncompressedSize;
         }
-        return hpatch_TRUE;
     }
 
     //file entry end

src/patch/Zipper.cpp

@@ -526,16 +526,19 @@ bool UnZipper_updateVirtualVCE(UnZipper* self,bool isDataNormalized,size_t zipCE
     return true;
 }
 
+//NOTE: used bit pos in generalPurposeBitFlag for saving isPageAlignSoFile tag
+#define kPurposeBitFlag_so_pos 7 //now 7,8?,10 unused by zip 
+#define _kPurposeBitFlag_so_BytePos (kPurposeBitFlag_so_pos/8)
+#define _kPurposeBitFlag_so_bitPos (kPurposeBitFlag_so_pos-(kPurposeBitFlag_so_pos/8)*8)
 
 inline static unsigned char* _at_file_generalPurposeBitFlag(const UnZipper* self,int fileIndex){
-    return fileHeaderBuf(self,fileIndex)+8;
+    return fileHeaderBuf(self,fileIndex)+8+_kPurposeBitFlag_so_BytePos;
 }
-//NOTE: used bit pos 7 in generalPurposeBitFlag for saving isPageAlignSoFile tag
 inline static bool _file_getIsPageAlignSoFile(const UnZipper* self,int fileIndex){
-    return 0!=((*_at_file_generalPurposeBitFlag(self,fileIndex))&(1<<7));
+    return 0!=((*_at_file_generalPurposeBitFlag(self,fileIndex))&(1<<_kPurposeBitFlag_so_bitPos));
 }
 inline static void _file_setIsPageAlignSoFile(const UnZipper* self,int fileIndex){
-    (*_at_file_generalPurposeBitFlag(self,fileIndex))|=(1<<7);
+    (*_at_file_generalPurposeBitFlag(self,fileIndex))|=(1<<_kPurposeBitFlag_so_bitPos);
 }
 
 inline static const unsigned char* _at_file_compressType(const UnZipper* self,int fileIndex){
@@ -860,13 +863,13 @@ bool Zipper_close(Zipper* self){
 
     self->_stream=0;
     self->_fileEntryCount=0;
-    if (self->_buf) { free(self->_buf); self->_buf=0; }
     if (self->_fileStream.m_file) { check(hpatch_TFileStreamOutput_close(&self->_fileStream)); }
     if (self->_append_stream.compressHandle!=0){
         struct _zlib_TCompress* compressHandle=self->_append_stream.compressHandle;
         self->_append_stream.compressHandle=0;
         check(_zlib_compress_close_by(compressPlugin,compressHandle));
     }
+    if (self->_buf) { free(self->_buf); self->_buf=0; }
     return true;
 }
 
@@ -998,7 +1001,6 @@ inline static size_t _getAlignSkipLen(size_t curPos,size_t align){
     return align-1-(curPos+align-1)%align;
 }
 inline static bool _writeAlignSkip(Zipper* self,size_t alignSkipLen){
-    assert(alignSkipLen<self->_ZipAlignSize);
     const size_t bufSize =16;
     const TByte _alignSkipBuf[bufSize]={0};
     while (alignSkipLen>0) {
@@ -1289,12 +1291,15 @@ bool Zipper_file_append_end(Zipper* self){
     if (append_state->self==0) { assert(false); return false; }
 
     bool result=true;
-    if (append_state->compressHandle!=0){
+    const bool isCompressedFile=(append_state->compressHandle!=0);
+    if (isCompressedFile){
         if ((append_state->inputPos==0)&&(append_state->outputPos==0)){
             Byte emptyBuf=0; //compress empty file
             check(append_state->write(append_state,0,&emptyBuf,&emptyBuf));
         }
-        check_clear(_zlib_compress_close_by(compressPlugin,append_state->compressHandle));
+        struct _zlib_TCompress* compressHandle=append_state->compressHandle;
+        append_state->compressHandle=0;
+        check_clear(_zlib_compress_close_by(compressPlugin,compressHandle));
     }
 
     check_clear(append_state->inputPos==append_state->streamSize);
@@ -1307,7 +1312,7 @@ bool Zipper_file_append_end(Zipper* self){
         check_clear(_dispose_filishedThreadWork(self,false));
     }
 #endif
-    if (append_state->compressHandle!=0){
+    if (isCompressedFile){
         assert(append_state->outputPos==(uint32_t)append_state->outputPos);
         uint32_t compressedSize=(uint32_t)append_state->outputPos;
         check_clear(_zipper_file_update_compressedSize(self,append_state->curFileIndex,compressedSize));

src/patch/patch_types.h

@@ -33,7 +33,7 @@
 
 #define APKDIFFPATCH_VERSION_MAJOR    1
 #define APKDIFFPATCH_VERSION_MINOR    7
-#define APKDIFFPATCH_VERSION_RELEASE  0
+#define APKDIFFPATCH_VERSION_RELEASE  1
 
 #define _APKDIFFPATCH_VERSION       APKDIFFPATCH_VERSION_MAJOR.APKDIFFPATCH_VERSION_MINOR.APKDIFFPATCH_VERSION_RELEASE
 #define _APKDIFFPATCH_QUOTE(str)    #str