diff --git a/src/compress.cpp b/src/compress.cpp index 575b3191..554acf0b 100644 --- a/src/compress.cpp +++ b/src/compress.cpp @@ -87,6 +87,7 @@ int upx_compress ( const upx_bytep src, upx_uint src_len, result[5] = src_len; // max_run_found result[6] = 1; // first_offset_found //result[7] = 999999; // same_match_offsets_found - NOT USED + result[8] = 0; #if defined(WITH_LZMA) if (M_IS_LZMA(method)) @@ -147,6 +148,10 @@ int upx_test_overlap ( const upx_bytep buf, upx_uint src_off, { int r = UPX_E_ERROR; + assert(src_len < *dst_len); // must be compressed + unsigned overlap_overhead = src_off + src_len - *dst_len; + assert((int)overlap_overhead > 0); + #if defined(WITH_LZMA) if (M_IS_LZMA(method)) return upx_lzma_test_overlap(buf, src_off, src_len, dst_len, method); diff --git a/src/compress_lzma.cpp b/src/compress_lzma.cpp index 9fbb5403..353491d8 100644 --- a/src/compress_lzma.cpp +++ b/src/compress_lzma.cpp @@ -42,14 +42,14 @@ int compress_lzma_dummy = 0; #undef OS2 #undef _WIN32 #undef _WIN32_WCE -#include "lzma/MyInitGuid.h" -//#include "lzma/LZMADecoder.h" -#include "lzma/LZMAEncoder.h" -#undef RC_NORMALIZE +#undef COMPRESS_MF_MT +#include "C/Common/MyInitGuid.h" +#include "C/7zip/Compress/LZMA/LZMADecoder.h" +#include "C/7zip/Compress/LZMA/LZMAEncoder.h" namespace MyLzma { -struct CInStreamRam: public ISequentialInStream, public CMyUnknownImp +struct InStreamRam: public ISequentialInStream, public CMyUnknownImp { MY_UNKNOWN_IMP const Byte *Data; size_t Size; size_t Pos; @@ -59,7 +59,7 @@ struct CInStreamRam: public ISequentialInStream, public CMyUnknownImp STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize); }; -STDMETHODIMP CInStreamRam::Read(void *data, UInt32 size, UInt32 *processedSize) +STDMETHODIMP InStreamRam::Read(void *data, UInt32 size, UInt32 *processedSize) { UInt32 remain = Size - Pos; if (size > remain) size = remain; @@ -69,7 +69,7 @@ STDMETHODIMP CInStreamRam::Read(void *data, UInt32 size, UInt32 *processedSize) return S_OK; } -struct COutStreamRam : public ISequentialOutStream, public CMyUnknownImp +struct OutStreamRam : public ISequentialOutStream, public CMyUnknownImp { MY_UNKNOWN_IMP Byte *Data; size_t Size; size_t Pos; bool Overflow; @@ -84,7 +84,7 @@ struct COutStreamRam : public ISequentialOutStream, public CMyUnknownImp STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize); }; -STDMETHODIMP COutStreamRam::Write(const void *data, UInt32 size, UInt32 *processedSize) +STDMETHODIMP OutStreamRam::Write(const void *data, UInt32 size, UInt32 *processedSize) { UInt32 i; for (i = 0; i < size && Pos < Size; i++) @@ -94,6 +94,21 @@ STDMETHODIMP COutStreamRam::Write(const void *data, UInt32 size, UInt32 *process return S_OK; } + +struct ProgressInfo : public ICompressProgressInfo, public CMyUnknownImp +{ + MY_UNKNOWN_IMP + STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); + upx_callback_p cb; +}; + +STDMETHODIMP ProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize) +{ + if (cb && cb->nprogress) + cb->nprogress(cb, (upx_uint) *inSize, (upx_uint) *outSize, 3); + return S_OK; +} + } // namespace @@ -101,7 +116,7 @@ int upx_lzma_compress ( const upx_bytep src, upx_uint src_len, upx_bytep dst, upx_uintp dst_len, upx_callback_p cb, int method, int level, - const struct upx_compress_config_t *conf, + const struct upx_compress_config_t *conf_parm, upx_uintp result ) { assert(method == M_LZMA); @@ -111,41 +126,59 @@ int upx_lzma_compress ( const upx_bytep src, upx_uint src_len, int r = UPX_E_ERROR; HRESULT rh; - MyLzma::CInStreamRam* isp = new MyLzma::CInStreamRam; - MyLzma::CInStreamRam& is = *isp; - MyLzma::COutStreamRam* osp = new MyLzma::COutStreamRam; - MyLzma::COutStreamRam& os = *osp; + MyLzma::InStreamRam is; is.AddRef(); + MyLzma::OutStreamRam os; os.AddRef(); is.Init(src, src_len); // os.Init(dst, *dst_len); os.Init(dst, src_len); + MyLzma::ProgressInfo progress; progress.AddRef(); + progress.cb = cb; + #ifndef _NO_EXCEPTIONS try { #endif NCompress::NLZMA::CEncoder enc; - PROPID propIDs[3] = { - NCoderPropID::kAlgorithm, - NCoderPropID::kDictionarySize, - NCoderPropID::kNumFastBytes + const PROPID propIDs[7] = { + NCoderPropID::kPosStateBits, // 0 pb _posStateBits(2) + NCoderPropID::kLitPosBits, // 1 lp _numLiteralPosStateBits(0) + NCoderPropID::kLitContextBits, // 2 lc _numLiteralContextBits(3) + NCoderPropID::kDictionarySize, // 3 + NCoderPropID::kAlgorithm, // 4 _fastmode + NCoderPropID::kNumFastBytes, // 5 + NCoderPropID::kMatchFinderCycles // 6 }; - PROPVARIANT properties[3]; - properties[0].vt = VT_UI4; - properties[1].vt = VT_UI4; - properties[2].vt = VT_UI4; - properties[0].ulVal = (UInt32) 2; -// properties[1].ulVal = (UInt32) dictionarySize; - properties[1].ulVal = (UInt32) 1024 * 1024; // FIXME - properties[2].ulVal = (UInt32) 64; + PROPVARIANT pr[7]; + pr[0].vt = pr[1].vt = pr[2].vt = VT_UI4; + pr[3].vt = pr[4].vt = pr[5].vt = VT_UI4; + pr[6].vt = VT_UI4; - if (enc.SetCoderProperties(propIDs, properties, 3) != S_OK) + // setup defaults + pr[0].uintVal = 2; + pr[1].uintVal = 0; + pr[2].uintVal = 3; + pr[3].uintVal = src_len; + pr[4].uintVal = 2; + pr[5].uintVal = 64; + pr[6].uintVal = 0; + + // FIXME: tune these according to level + if (pr[3].uintVal > src_len) + pr[3].uintVal = src_len; + + if (enc.SetCoderProperties(propIDs, pr, 3) != S_OK) goto error; if (enc.WriteCoderProperties(&os) != S_OK) goto error; - if (os.Pos != 5) + if (os.Overflow || os.Pos != 5) goto error; +#if 0 // defined(_LZMA_OUT_READ) +#else + os.Pos -= 4; // do not encode dict_size +#endif - rh = enc.Code(&is, &os, 0, 0, 0); + rh = enc.Code(&is, &os, NULL, NULL, &progress); if (rh == E_OUTOFMEMORY) r = UPX_E_OUT_OF_MEMORY; #if 0 @@ -156,7 +189,8 @@ int upx_lzma_compress ( const upx_bytep src, upx_uint src_len, r = UPX_E_OK; //result[8] = LzmaGetNumProbs(&s.Properties)); - result[8] = 0; // FIXME + //result[8] = (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((Properties)->lc + (Properties)->lp))) + result[8] = 1846 + (768 << (pr[2].uintVal + pr[1].uintVal)); error: *dst_len = os.Pos; @@ -168,14 +202,17 @@ error: } -#include "lzma/Alloc.cpp" -#include "lzma/CRC.cpp" -//#include "lzma/LZMADecoder.cpp" -#include "lzma/LZMAEncoder.cpp" -#include "lzma/LZInWindow.cpp" -#include "lzma/OutBuffer.cpp" -#include "lzma/RangeCoderBit.cpp" -#include "lzma/StreamUtils.cpp" +#include "C/Common/Alloc.cpp" +#include "C/Common/CRC.cpp" +//#include "C/7zip/Common/InBuffer.cpp" +#include "C/7zip/Common/OutBuffer.cpp" +#include "C/7zip/Common/StreamUtils.cpp" +#include "C/7zip/Compress/LZ/LZInWindow.cpp" +//#include "C/7zip/Compress/LZ/LZOutWindow.cpp" +//#include "C/7zip/Compress/LZMA/LZMADecoder.cpp" +#include "C/7zip/Compress/LZMA/LZMAEncoder.cpp" +#include "C/7zip/Compress/RangeCoder/RangeCoderBit.cpp" +#undef RC_NORMALIZE /************************************************************************* @@ -186,7 +223,7 @@ error: #undef _LZMA_OUT_READ #undef _LZMA_PROB32 #undef _LZMA_LOC_OPT -#include "lzma/LzmaDecode.cpp" +#include "C/7zip/Compress/LZMA_C/LzmaDecode.c" int upx_lzma_decompress ( const upx_bytep src, upx_uint src_len, upx_bytep dst, upx_uintp dst_len, @@ -205,7 +242,11 @@ int upx_lzma_decompress ( const upx_bytep src, upx_uint src_len, r = LzmaDecodeProperties(&s.Properties, src, src_len); if (r != 0) goto error; +#if 0 // defined(_LZMA_OUT_READ) src += LZMA_PROPERTIES_SIZE; src_len -= LZMA_PROPERTIES_SIZE; +#else + src += 1; src_len -= 1; +#endif s.Probs = (CProb *) malloc(sizeof(CProb) * LzmaGetNumProbs(&s.Properties)); if (!s.Probs) r = UPX_E_OUT_OF_MEMORY; @@ -218,6 +259,28 @@ error: } +/************************************************************************* +// +**************************************************************************/ + +int upx_lzma_test_overlap ( const upx_bytep buf, upx_uint src_off, + upx_uint src_len, upx_uintp dst_len, + int method ) +{ + assert(method == M_LZMA); + + /* FIXME */ + UNUSED(buf); + + unsigned overlap_overhead = src_off + src_len - *dst_len; + //printf("upx_lzma_test_overlap: %d\n", overlap_overhead); + if ((int)overlap_overhead >= 256) + return UPX_E_OK; + + return UPX_E_ERROR; +} + + #endif /* WITH_LZMA */ /* vi:ts=4:et:nowrap diff --git a/src/conf.h b/src/conf.h index 2f7828ef..2e50c5cb 100644 --- a/src/conf.h +++ b/src/conf.h @@ -144,6 +144,9 @@ # undef __unix #endif +#if 1 && !defined(WITH_LZMA) +# define WITH_LZMA 1 +#endif #if !defined(WITH_UCL) # define WITH_UCL 1 #endif @@ -181,7 +184,6 @@ #endif struct upx_callback_t; -typedef struct upx_callback_t upx_callback_t; #define upx_callback_p upx_callback_t * typedef upx_voidp (__UPX_CDECL *upx_alloc_func_t) (upx_callback_p self, upx_uint items, upx_uint size); @@ -200,10 +202,20 @@ struct upx_callback_t upx_uint user3; }; +struct lzma_compress_config_t +{ + unsigned pos_bits; + unsigned lit_pos_bits; + unsigned lit_context_bits; + unsigned dict_size; + unsigned mf_passes; +}; + #define upx_compress_config_p upx_compress_config_t * struct upx_compress_config_t { - ucl_compress_config_t conf_ucl; + lzma_compress_config_t conf_lzma; + ucl_compress_config_t conf_ucl; }; @@ -441,6 +453,8 @@ inline void operator delete[](void *p) // compression methods - DO NOT CHANGE +#define M_SKIP (-2) +#define M_END (-1) #define M_NRV2B_LE32 2 #define M_NRV2B_8 3 #define M_NRV2B_LE16 4 diff --git a/src/main.cpp b/src/main.cpp index 3014efb3..eb3cf318 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -512,6 +512,7 @@ static int do_option(int optc, const char *arg) e_method(M_NRV2E_LE32, opt->level); break; case 721: + opt->all_methods_use_lzma = true; if (!set_method(M_LZMA, -1)) e_method(M_LZMA, opt->level); break; @@ -532,6 +533,7 @@ static int do_option(int optc, const char *arg) case 901: // --brute opt->all_methods = true; + opt->all_methods_use_lzma = true; opt->method = -1; opt->all_filters = true; opt->filter = -1; diff --git a/src/options.h b/src/options.h index 9fb0ff06..3d4062e9 100644 --- a/src/options.h +++ b/src/options.h @@ -47,11 +47,12 @@ struct options_t { // compression options int method; - int level; // compression level 1..10 - int filter; // preferred filter from Packer::getFilters() - bool all_methods; // try all available compression methods ? - bool all_filters; // try all available filters ? - bool no_filter; // force no filter + int level; // compression level 1..10 + int filter; // preferred filter from Packer::getFilters() + bool all_methods; // try all available compression methods ? + bool all_methods_use_lzma; + bool all_filters; // try all available filters ? + bool no_filter; // force no filter // other options int backup; diff --git a/src/p_lx_elf.cpp b/src/p_lx_elf.cpp index 84e91731..6bccc8d7 100644 --- a/src/p_lx_elf.cpp +++ b/src/p_lx_elf.cpp @@ -195,8 +195,8 @@ int const * PackLinuxElf64amd::getCompressionMethods(int method, int level) const { // No real dependency on LE32. - static const int m_nrv2b[] = { M_NRV2B_LE32, M_NRV2E_LE32, -1 }; - static const int m_nrv2e[] = { M_NRV2E_LE32, M_NRV2B_LE32, -1 }; + static const int m_nrv2b[] = { M_NRV2B_LE32, M_NRV2E_LE32, M_LZMA, -1 }; + static const int m_nrv2e[] = { M_NRV2E_LE32, M_NRV2B_LE32, M_LZMA, -1 }; static const int m_lzma[] = { M_LZMA,-1 }; if (M_IS_NRV2B(method)) @@ -1534,11 +1534,12 @@ void PackLinuxElf64::unpack(OutputFile *fo) throwChecksumError(); #undef MAX_ELF_HDR } + + /************************************************************************* // **************************************************************************/ - PackLinuxElf32x86::PackLinuxElf32x86(InputFile *f) : super(f) { } diff --git a/src/packer.cpp b/src/packer.cpp index 3fe3673e..9654443f 100644 --- a/src/packer.cpp +++ b/src/packer.cpp @@ -138,6 +138,21 @@ bool Packer::testUnpackFormat(int format) const } +bool Packer::skipVerify(int method, int level) const +{ + if (method == M_LZMA) + return false; + if (level > 1) + return false; + return true; +} + +bool Packer::skipVerify() const +{ + return skipVerify(ph.method, ph.level); +} + + /************************************************************************* // compress **************************************************************************/ @@ -228,7 +243,7 @@ bool Packer::compress(upx_bytep in, upx_bytep out, // update checksum of compressed data ph.c_adler = upx_adler32(out, ph.c_len, ph.c_adler); // Decompress and verify. Skip this when using the fastest level. - if (ph.level > 1) + if (!skipVerify()) { // decompress unsigned new_len = ph.u_len; @@ -332,12 +347,16 @@ bool Packer::testOverlappingDecompression(const upx_bytep buf, return false; assert((int)overlap_overhead >= 0); + // Because upx_test_overlap() does not use the asm_fast decompressor // we must account for extra 3 bytes that asm_fast does use, // or else we may fail at runtime decompression. - if (overlap_overhead <= 4 + 3) // don't waste time here + unsigned extra = 0; + if (M_IS_NRV2B(ph.method) || M_IS_NRV2D(ph.method) || M_IS_NRV2E(ph.method)) + extra = 3; + if (overlap_overhead <= 4 + extra) // don't waste time here return false; - overlap_overhead -= 3; + overlap_overhead -= extra; unsigned src_off = ph.u_len + overlap_overhead - ph.c_len; unsigned new_len = ph.u_len; @@ -366,7 +385,7 @@ void Packer::verifyOverlappingDecompression(Filter *ft) // See also: // Filter::verifyUnfilter() - if (ph.level == 1) + if (skipVerify()) return; unsigned offset = (ph.u_len + ph.overlap_overhead) - ph.c_len; if (offset + ph.c_len > obuf.getSize()) @@ -407,7 +426,9 @@ unsigned Packer::findOverlapOverhead(const upx_bytep buf, assert(m >= low); assert(m <= high); assert(m < overhead || overhead == 0); nr++; - if (testOverlappingDecompression(buf, m)) + bool success = testOverlappingDecompression(buf, m); + //printf("testOverlapOverhead: %d %d -> %d\n", nr, m, (int)success); + if (success) { overhead = m; // Succeed early if m lies in [low .. low+range-1], i.e. if @@ -920,17 +941,22 @@ unsigned Packer::unoptimizeReloc32(upx_byte **in, upx_byte *image, bool Packer::isValidCompressionMethod(int method) { +#if !defined(WITH_LZMA) + if (method == M_LZMA) { + assert(0 && "Internal error - LZMA not compiled in"); + } +#endif return (method >= M_NRV2B_LE32 && method <= M_LZMA); } const int *Packer::getDefaultCompressionMethods_8(int method, int level, int small) const { - static const int m_nrv2b[] = { M_NRV2B_8, M_NRV2D_8, M_NRV2E_8, -1 }; - static const int m_nrv2d[] = { M_NRV2D_8, M_NRV2B_8, M_NRV2E_8, -1 }; - static const int m_nrv2e[] = { M_NRV2E_8, M_NRV2B_8, M_NRV2D_8, -1 }; - static const int m_cl1b[] = { M_CL1B_8, -1 }; - static const int m_lzma[] = { M_LZMA, -1 }; + static const int m_nrv2b[] = { M_NRV2B_8, M_NRV2D_8, M_NRV2E_8, M_LZMA, M_END }; + static const int m_nrv2d[] = { M_NRV2D_8, M_NRV2B_8, M_NRV2E_8, M_LZMA, M_END }; + static const int m_nrv2e[] = { M_NRV2E_8, M_NRV2B_8, M_NRV2D_8, M_LZMA, M_END }; + static const int m_cl1b[] = { M_CL1B_8, M_END }; + static const int m_lzma[] = { M_LZMA, M_END }; if (small < 0) small = file_size <= 512*1024; @@ -944,6 +970,7 @@ const int *Packer::getDefaultCompressionMethods_8(int method, int level, int sma return m_cl1b; if (M_IS_LZMA(method)) return m_lzma; + assert(method == -1); // --all-methods if (level == 1 || small) return m_nrv2b; return m_nrv2e; @@ -952,11 +979,11 @@ const int *Packer::getDefaultCompressionMethods_8(int method, int level, int sma const int *Packer::getDefaultCompressionMethods_le32(int method, int level, int small) const { - static const int m_nrv2b[] = { M_NRV2B_LE32, M_NRV2D_LE32, M_NRV2E_LE32, -1 }; - static const int m_nrv2d[] = { M_NRV2D_LE32, M_NRV2B_LE32, M_NRV2E_LE32, -1 }; - static const int m_nrv2e[] = { M_NRV2E_LE32, M_NRV2B_LE32, M_NRV2D_LE32, -1 }; - static const int m_cl1b[] = { M_CL1B_LE32, -1 }; - static const int m_lzma[] = { M_LZMA, -1 }; + static const int m_nrv2b[] = { M_NRV2B_LE32, M_NRV2D_LE32, M_NRV2E_LE32, M_LZMA, M_END }; + static const int m_nrv2d[] = { M_NRV2D_LE32, M_NRV2B_LE32, M_NRV2E_LE32, M_LZMA, M_END }; + static const int m_nrv2e[] = { M_NRV2E_LE32, M_NRV2B_LE32, M_NRV2D_LE32, M_LZMA, M_END }; + static const int m_cl1b[] = { M_CL1B_LE32, M_END }; + static const int m_lzma[] = { M_LZMA, M_END }; if (small < 0) small = file_size <= 512*1024; @@ -970,6 +997,7 @@ const int *Packer::getDefaultCompressionMethods_le32(int method, int level, int return m_cl1b; if (M_IS_LZMA(method)) return m_lzma; + assert(method == -1); // --all-methods if (level == 1 || small) return m_nrv2b; return m_nrv2e; @@ -1363,16 +1391,20 @@ void Packer::compressWithFilters(Filter *parm_ft, filters[nfilters] = -1; // methods - int tmp_methods[] = { ph.method, -1 }; + int tmp_methods[] = { ph.method, M_END }; const int *methods = NULL; if (opt->all_methods) methods = getCompressionMethods(-1, ph.level); if (methods == NULL) methods = tmp_methods; int nmethods = 0; - while (methods[nmethods] >= 0) + for (int mm = 0; methods[mm] != M_END; ++mm) { - assert(isValidCompressionMethod(methods[nmethods])); + if (methods[mm] == M_SKIP) + continue; + if (opt->all_methods && !opt->all_methods_use_lzma && methods[mm] == M_LZMA) + continue; + assert(isValidCompressionMethod(methods[mm])); nmethods++; } assert(nmethods > 0); @@ -1389,17 +1421,23 @@ void Packer::compressWithFilters(Filter *parm_ft, // compress int nfilters_success = 0; - for (int m = 0; m < nmethods; m++) // for all methods + for (int mm = 0; methods[mm] != M_END; ++mm) // for all methods { + if (methods[mm] == M_SKIP) + continue; + if (opt->all_methods && !opt->all_methods_use_lzma && methods[mm] == M_LZMA) + continue; unsigned hdr_c_len = 0; if (hdr_buf && hdr_u_len) { - if (0 < m && otemp == &obuf) { // do not overwrite obuf + if (nfilters_success != 0 && otemp == &obuf) + { + // do not overwrite obuf otemp_buf.allocForCompression(compress_buf_len); otemp = &otemp_buf; } int r = upx_compress(hdr_buf, hdr_u_len, *otemp, &hdr_c_len, - NULL, methods[m], 10, NULL, NULL); + NULL, methods[mm], 10, NULL, NULL); if (r != UPX_E_OK) throwInternalError("header compression failed"); if (hdr_c_len >= hdr_u_len) @@ -1411,7 +1449,7 @@ void Packer::compressWithFilters(Filter *parm_ft, obuf.checkState(); // get fresh packheader ph = orig_ph; - ph.method = methods[m]; + ph.method = methods[mm]; ph.filter = filters[i]; ph.overlap_overhead = 0; // get fresh filter @@ -1442,7 +1480,7 @@ void Packer::compressWithFilters(Filter *parm_ft, printf("filter: id 0x%02x size %6d, calls %5d/%5d/%3d/%5d/%5d, cto 0x%02x\n", ft.id, ft.buf_len, ft.calls, ft.noncalls, ft.wrongcalls, ft.firstcall, ft.lastcall, ft.cto); #endif - if (nfilters_success > 0 && otemp == &obuf) + if (nfilters_success != 0 && otemp == &obuf) { otemp_buf.allocForCompression(compress_buf_len); otemp = &otemp_buf; diff --git a/src/packer.h b/src/packer.h index 9dc8af57..29d86bea 100644 --- a/src/packer.h +++ b/src/packer.h @@ -137,6 +137,9 @@ protected: // unpacker tests - these may throw exceptions virtual bool testUnpackVersion(int version) const; virtual bool testUnpackFormat(int format) const; + // + virtual bool skipVerify(int method, int level) const; + virtual bool skipVerify() const; protected: // implementation