mirror/upx
1
0
Fork 0
mirror of https://github.com/upx/upx synced 2025-09-28 19:06:07 +08:00
Code Issues Releases Wiki Activity

More lzma stuff.

Browse Source
This commit is contained in:
Markus F.X.J. Oberhumer 2006-06-09 18:36:53 +02:00
parent 54340ab4bf
commit f1ba0c0aa7
8 changed files with 198 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/compress.cpp
View File

@ -87,6 +87,7 @@ int upx_compress ( const upx_bytep src, upx_uint src_len,
result[5] = src_len; // max_run_found
result[6] = 1; // first_offset_found
//result[7] = 999999; // same_match_offsets_found - NOT USED
result[8] = 0;
#if defined(WITH_LZMA)
if (M_IS_LZMA(method))
@ -147,6 +148,10 @@ int upx_test_overlap ( const upx_bytep buf, upx_uint src_off,
{
int r = UPX_E_ERROR;
assert(src_len < *dst_len); // must be compressed
unsigned overlap_overhead = src_off + src_len - *dst_len;
assert((int)overlap_overhead > 0);
#if defined(WITH_LZMA)
if (M_IS_LZMA(method))
return upx_lzma_test_overlap(buf, src_off, src_len, dst_len, method);

139
src/compress_lzma.cpp
View File

@ -42,14 +42,14 @@ int compress_lzma_dummy = 0;
#undef OS2
#undef _WIN32
#undef _WIN32_WCE
#include "lzma/MyInitGuid.h"
//#include "lzma/LZMADecoder.h"
#include "lzma/LZMAEncoder.h"
#undef RC_NORMALIZE
#undef COMPRESS_MF_MT
#include "C/Common/MyInitGuid.h"
#include "C/7zip/Compress/LZMA/LZMADecoder.h"
#include "C/7zip/Compress/LZMA/LZMAEncoder.h"
namespace MyLzma {
struct CInStreamRam: public ISequentialInStream, public CMyUnknownImp
struct InStreamRam: public ISequentialInStream, public CMyUnknownImp
{
MY_UNKNOWN_IMP
const Byte *Data; size_t Size; size_t Pos;
@ -59,7 +59,7 @@ struct CInStreamRam: public ISequentialInStream, public CMyUnknownImp
STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
};
STDMETHODIMP CInStreamRam::Read(void *data, UInt32 size, UInt32 *processedSize)
STDMETHODIMP InStreamRam::Read(void *data, UInt32 size, UInt32 *processedSize)
{
UInt32 remain = Size - Pos;
if (size > remain) size = remain;
@ -69,7 +69,7 @@ STDMETHODIMP CInStreamRam::Read(void *data, UInt32 size, UInt32 *processedSize)
return S_OK;
}
struct COutStreamRam : public ISequentialOutStream, public CMyUnknownImp
struct OutStreamRam : public ISequentialOutStream, public CMyUnknownImp
{
MY_UNKNOWN_IMP
Byte *Data; size_t Size; size_t Pos; bool Overflow;
@ -84,7 +84,7 @@ struct COutStreamRam : public ISequentialOutStream, public CMyUnknownImp
STDMETHOD(Write)(const void *data, UInt32 size, UInt32 *processedSize);
};
STDMETHODIMP COutStreamRam::Write(const void *data, UInt32 size, UInt32 *processedSize)
STDMETHODIMP OutStreamRam::Write(const void *data, UInt32 size, UInt32 *processedSize)
{
UInt32 i;
for (i = 0; i < size && Pos < Size; i++)
@ -94,6 +94,21 @@ STDMETHODIMP COutStreamRam::Write(const void *data, UInt32 size, UInt32 *process
return S_OK;
}
struct ProgressInfo : public ICompressProgressInfo, public CMyUnknownImp
{
MY_UNKNOWN_IMP
STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize);
upx_callback_p cb;
};
STDMETHODIMP ProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize)
{
if (cb && cb->nprogress)
cb->nprogress(cb, (upx_uint) *inSize, (upx_uint) *outSize, 3);
return S_OK;
}
} // namespace
@ -101,7 +116,7 @@ int upx_lzma_compress ( const upx_bytep src, upx_uint src_len,
upx_bytep dst, upx_uintp dst_len,
upx_callback_p cb,
int method, int level,
const struct upx_compress_config_t *conf,
const struct upx_compress_config_t *conf_parm,
upx_uintp result )
{
assert(method == M_LZMA);
@ -111,41 +126,59 @@ int upx_lzma_compress ( const upx_bytep src, upx_uint src_len,
int r = UPX_E_ERROR;
HRESULT rh;
MyLzma::CInStreamRam* isp = new MyLzma::CInStreamRam;
MyLzma::CInStreamRam& is = *isp;
MyLzma::COutStreamRam* osp = new MyLzma::COutStreamRam;
MyLzma::COutStreamRam& os = *osp;
MyLzma::InStreamRam is; is.AddRef();
MyLzma::OutStreamRam os; os.AddRef();
is.Init(src, src_len);
// os.Init(dst, *dst_len);
os.Init(dst, src_len);
MyLzma::ProgressInfo progress; progress.AddRef();
progress.cb = cb;
#ifndef _NO_EXCEPTIONS
try {
#endif
NCompress::NLZMA::CEncoder enc;
PROPID propIDs[3] = {
NCoderPropID::kAlgorithm,
NCoderPropID::kDictionarySize,
NCoderPropID::kNumFastBytes
const PROPID propIDs[7] = {
NCoderPropID::kPosStateBits, // 0 pb _posStateBits(2)
NCoderPropID::kLitPosBits, // 1 lp _numLiteralPosStateBits(0)
NCoderPropID::kLitContextBits, // 2 lc _numLiteralContextBits(3)
NCoderPropID::kDictionarySize, // 3
NCoderPropID::kAlgorithm, // 4 _fastmode
NCoderPropID::kNumFastBytes, // 5
NCoderPropID::kMatchFinderCycles // 6
};
PROPVARIANT properties[3];
properties[0].vt = VT_UI4;
properties[1].vt = VT_UI4;
properties[2].vt = VT_UI4;
properties[0].ulVal = (UInt32) 2;
// properties[1].ulVal = (UInt32) dictionarySize;
properties[1].ulVal = (UInt32) 1024 * 1024; // FIXME
properties[2].ulVal = (UInt32) 64;
PROPVARIANT pr[7];
pr[0].vt = pr[1].vt = pr[2].vt = VT_UI4;
pr[3].vt = pr[4].vt = pr[5].vt = VT_UI4;
pr[6].vt = VT_UI4;
if (enc.SetCoderProperties(propIDs, properties, 3) != S_OK)
// setup defaults
pr[0].uintVal = 2;
pr[1].uintVal = 0;
pr[2].uintVal = 3;
pr[3].uintVal = src_len;
pr[4].uintVal = 2;
pr[5].uintVal = 64;
pr[6].uintVal = 0;
// FIXME: tune these according to level
if (pr[3].uintVal > src_len)
pr[3].uintVal = src_len;
if (enc.SetCoderProperties(propIDs, pr, 3) != S_OK)
goto error;
if (enc.WriteCoderProperties(&os) != S_OK)
goto error;
if (os.Pos != 5)
if (os.Overflow || os.Pos != 5)
goto error;
#if 0 // defined(_LZMA_OUT_READ)
#else
os.Pos -= 4; // do not encode dict_size
#endif
rh = enc.Code(&is, &os, 0, 0, 0);
rh = enc.Code(&is, &os, NULL, NULL, &progress);
if (rh == E_OUTOFMEMORY)
r = UPX_E_OUT_OF_MEMORY;
#if 0
@ -156,7 +189,8 @@ int upx_lzma_compress ( const upx_bytep src, upx_uint src_len,
r = UPX_E_OK;
//result[8] = LzmaGetNumProbs(&s.Properties));
result[8] = 0; // FIXME
//result[8] = (LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((Properties)->lc + (Properties)->lp)))
result[8] = 1846 + (768 << (pr[2].uintVal + pr[1].uintVal));
error:
*dst_len = os.Pos;
@ -168,14 +202,17 @@ error:
}
#include "lzma/Alloc.cpp"
#include "lzma/CRC.cpp"
//#include "lzma/LZMADecoder.cpp"
#include "lzma/LZMAEncoder.cpp"
#include "lzma/LZInWindow.cpp"
#include "lzma/OutBuffer.cpp"
#include "lzma/RangeCoderBit.cpp"
#include "lzma/StreamUtils.cpp"
#include "C/Common/Alloc.cpp"
#include "C/Common/CRC.cpp"
//#include "C/7zip/Common/InBuffer.cpp"
#include "C/7zip/Common/OutBuffer.cpp"
#include "C/7zip/Common/StreamUtils.cpp"
#include "C/7zip/Compress/LZ/LZInWindow.cpp"
//#include "C/7zip/Compress/LZ/LZOutWindow.cpp"
//#include "C/7zip/Compress/LZMA/LZMADecoder.cpp"
#include "C/7zip/Compress/LZMA/LZMAEncoder.cpp"
#include "C/7zip/Compress/RangeCoder/RangeCoderBit.cpp"
#undef RC_NORMALIZE
/*************************************************************************
@ -186,7 +223,7 @@ error:
#undef _LZMA_OUT_READ
#undef _LZMA_PROB32
#undef _LZMA_LOC_OPT
#include "lzma/LzmaDecode.cpp"
#include "C/7zip/Compress/LZMA_C/LzmaDecode.c"
int upx_lzma_decompress ( const upx_bytep src, upx_uint src_len,
upx_bytep dst, upx_uintp dst_len,
@ -205,7 +242,11 @@ int upx_lzma_decompress ( const upx_bytep src, upx_uint src_len,
r = LzmaDecodeProperties(&s.Properties, src, src_len);
if (r != 0)
goto error;
#if 0 // defined(_LZMA_OUT_READ)
src += LZMA_PROPERTIES_SIZE; src_len -= LZMA_PROPERTIES_SIZE;
#else
src += 1; src_len -= 1;
#endif
s.Probs = (CProb *) malloc(sizeof(CProb) * LzmaGetNumProbs(&s.Properties));
if (!s.Probs)
r = UPX_E_OUT_OF_MEMORY;
@ -218,6 +259,28 @@ error:
}
/*************************************************************************
//
**************************************************************************/
int upx_lzma_test_overlap ( const upx_bytep buf, upx_uint src_off,
upx_uint src_len, upx_uintp dst_len,
int method )
{
assert(method == M_LZMA);
/* FIXME */
UNUSED(buf);
unsigned overlap_overhead = src_off + src_len - *dst_len;
//printf("upx_lzma_test_overlap: %d\n", overlap_overhead);
if ((int)overlap_overhead >= 256)
return UPX_E_OK;
return UPX_E_ERROR;
}
#endif /* WITH_LZMA */
/*
vi:ts=4:et:nowrap

18
src/conf.h
View File

@ -144,6 +144,9 @@
# undef __unix
#endif
#if 1 && !defined(WITH_LZMA)
# define WITH_LZMA 1
#endif
#if !defined(WITH_UCL)
# define WITH_UCL 1
#endif
@ -181,7 +184,6 @@
#endif
struct upx_callback_t;
typedef struct upx_callback_t upx_callback_t;
#define upx_callback_p upx_callback_t *
typedef upx_voidp (__UPX_CDECL *upx_alloc_func_t)
(upx_callback_p self, upx_uint items, upx_uint size);
@ -200,10 +202,20 @@ struct upx_callback_t
upx_uint user3;
};
struct lzma_compress_config_t
{
unsigned pos_bits;
unsigned lit_pos_bits;
unsigned lit_context_bits;
unsigned dict_size;
unsigned mf_passes;
};
#define upx_compress_config_p upx_compress_config_t *
struct upx_compress_config_t
{
ucl_compress_config_t conf_ucl;
lzma_compress_config_t conf_lzma;
ucl_compress_config_t conf_ucl;
};
@ -441,6 +453,8 @@ inline void operator delete[](void *p)
// compression methods - DO NOT CHANGE
#define M_SKIP (-2)
#define M_END (-1)
#define M_NRV2B_LE32 2
#define M_NRV2B_8 3
#define M_NRV2B_LE16 4

2
src/main.cpp
View File

@ -512,6 +512,7 @@ static int do_option(int optc, const char *arg)
e_method(M_NRV2E_LE32, opt->level);
break;
case 721:
opt->all_methods_use_lzma = true;
if (!set_method(M_LZMA, -1))
e_method(M_LZMA, opt->level);
break;
@ -532,6 +533,7 @@ static int do_option(int optc, const char *arg)
case 901: // --brute
opt->all_methods = true;
opt->all_methods_use_lzma = true;
opt->method = -1;
opt->all_filters = true;
opt->filter = -1;

11
src/options.h
View File

@ -47,11 +47,12 @@ struct options_t {
// compression options
int method;
int level; // compression level 1..10
int filter; // preferred filter from Packer::getFilters()
bool all_methods; // try all available compression methods ?
bool all_filters; // try all available filters ?
bool no_filter; // force no filter
int level; // compression level 1..10
int filter; // preferred filter from Packer::getFilters()
bool all_methods; // try all available compression methods ?
bool all_methods_use_lzma;
bool all_filters; // try all available filters ?
bool no_filter; // force no filter
// other options
int backup;

7
src/p_lx_elf.cpp
View File

@ -195,8 +195,8 @@ int const *
PackLinuxElf64amd::getCompressionMethods(int method, int level) const
{
// No real dependency on LE32.
static const int m_nrv2b[] = { M_NRV2B_LE32, M_NRV2E_LE32, -1 };
static const int m_nrv2e[] = { M_NRV2E_LE32, M_NRV2B_LE32, -1 };
static const int m_nrv2b[] = { M_NRV2B_LE32, M_NRV2E_LE32, M_LZMA, -1 };
static const int m_nrv2e[] = { M_NRV2E_LE32, M_NRV2B_LE32, M_LZMA, -1 };
static const int m_lzma[] = { M_LZMA,-1 };
if (M_IS_NRV2B(method))
@ -1534,11 +1534,12 @@ void PackLinuxElf64::unpack(OutputFile *fo)
throwChecksumError();
#undef MAX_ELF_HDR
}
/*************************************************************************
//
**************************************************************************/
PackLinuxElf32x86::PackLinuxElf32x86(InputFile *f) : super(f)
{
}

84
src/packer.cpp
View File

@ -138,6 +138,21 @@ bool Packer::testUnpackFormat(int format) const
}
bool Packer::skipVerify(int method, int level) const
{
if (method == M_LZMA)
return false;
if (level > 1)
return false;
return true;
}
bool Packer::skipVerify() const
{
return skipVerify(ph.method, ph.level);
}
/*************************************************************************
// compress
**************************************************************************/
@ -228,7 +243,7 @@ bool Packer::compress(upx_bytep in, upx_bytep out,
// update checksum of compressed data
ph.c_adler = upx_adler32(out, ph.c_len, ph.c_adler);
// Decompress and verify. Skip this when using the fastest level.
if (ph.level > 1)
if (!skipVerify())
{
// decompress
unsigned new_len = ph.u_len;
@ -332,12 +347,16 @@ bool Packer::testOverlappingDecompression(const upx_bytep buf,
return false;
assert((int)overlap_overhead >= 0);
// Because upx_test_overlap() does not use the asm_fast decompressor
// we must account for extra 3 bytes that asm_fast does use,
// or else we may fail at runtime decompression.
if (overlap_overhead <= 4 + 3) // don't waste time here
unsigned extra = 0;
if (M_IS_NRV2B(ph.method) || M_IS_NRV2D(ph.method) || M_IS_NRV2E(ph.method))
extra = 3;
if (overlap_overhead <= 4 + extra) // don't waste time here
return false;
overlap_overhead -= 3;
overlap_overhead -= extra;
unsigned src_off = ph.u_len + overlap_overhead - ph.c_len;
unsigned new_len = ph.u_len;
@ -366,7 +385,7 @@ void Packer::verifyOverlappingDecompression(Filter *ft)
// See also:
// Filter::verifyUnfilter()
if (ph.level == 1)
if (skipVerify())
return;
unsigned offset = (ph.u_len + ph.overlap_overhead) - ph.c_len;
if (offset + ph.c_len > obuf.getSize())
@ -407,7 +426,9 @@ unsigned Packer::findOverlapOverhead(const upx_bytep buf,
assert(m >= low); assert(m <= high);
assert(m < overhead || overhead == 0);
nr++;
if (testOverlappingDecompression(buf, m))
bool success = testOverlappingDecompression(buf, m);
//printf("testOverlapOverhead: %d %d -> %d\n", nr, m, (int)success);
if (success)
{
overhead = m;
// Succeed early if m lies in [low .. low+range-1], i.e. if
@ -920,17 +941,22 @@ unsigned Packer::unoptimizeReloc32(upx_byte **in, upx_byte *image,
bool Packer::isValidCompressionMethod(int method)
{
#if !defined(WITH_LZMA)
if (method == M_LZMA) {
assert(0 && "Internal error - LZMA not compiled in");
}
#endif
return (method >= M_NRV2B_LE32 && method <= M_LZMA);
}
const int *Packer::getDefaultCompressionMethods_8(int method, int level, int small) const
{
static const int m_nrv2b[] = { M_NRV2B_8, M_NRV2D_8, M_NRV2E_8, -1 };
static const int m_nrv2d[] = { M_NRV2D_8, M_NRV2B_8, M_NRV2E_8, -1 };
static const int m_nrv2e[] = { M_NRV2E_8, M_NRV2B_8, M_NRV2D_8, -1 };
static const int m_cl1b[] = { M_CL1B_8, -1 };
static const int m_lzma[] = { M_LZMA, -1 };
static const int m_nrv2b[] = { M_NRV2B_8, M_NRV2D_8, M_NRV2E_8, M_LZMA, M_END };
static const int m_nrv2d[] = { M_NRV2D_8, M_NRV2B_8, M_NRV2E_8, M_LZMA, M_END };
static const int m_nrv2e[] = { M_NRV2E_8, M_NRV2B_8, M_NRV2D_8, M_LZMA, M_END };
static const int m_cl1b[] = { M_CL1B_8, M_END };
static const int m_lzma[] = { M_LZMA, M_END };
if (small < 0)
small = file_size <= 512*1024;
@ -944,6 +970,7 @@ const int *Packer::getDefaultCompressionMethods_8(int method, int level, int sma
return m_cl1b;
if (M_IS_LZMA(method))
return m_lzma;
assert(method == -1); // --all-methods
if (level == 1 || small)
return m_nrv2b;
return m_nrv2e;
@ -952,11 +979,11 @@ const int *Packer::getDefaultCompressionMethods_8(int method, int level, int sma
const int *Packer::getDefaultCompressionMethods_le32(int method, int level, int small) const
{
static const int m_nrv2b[] = { M_NRV2B_LE32, M_NRV2D_LE32, M_NRV2E_LE32, -1 };
static const int m_nrv2d[] = { M_NRV2D_LE32, M_NRV2B_LE32, M_NRV2E_LE32, -1 };
static const int m_nrv2e[] = { M_NRV2E_LE32, M_NRV2B_LE32, M_NRV2D_LE32, -1 };
static const int m_cl1b[] = { M_CL1B_LE32, -1 };
static const int m_lzma[] = { M_LZMA, -1 };
static const int m_nrv2b[] = { M_NRV2B_LE32, M_NRV2D_LE32, M_NRV2E_LE32, M_LZMA, M_END };
static const int m_nrv2d[] = { M_NRV2D_LE32, M_NRV2B_LE32, M_NRV2E_LE32, M_LZMA, M_END };
static const int m_nrv2e[] = { M_NRV2E_LE32, M_NRV2B_LE32, M_NRV2D_LE32, M_LZMA, M_END };
static const int m_cl1b[] = { M_CL1B_LE32, M_END };
static const int m_lzma[] = { M_LZMA, M_END };
if (small < 0)
small = file_size <= 512*1024;
@ -970,6 +997,7 @@ const int *Packer::getDefaultCompressionMethods_le32(int method, int level, int
return m_cl1b;
if (M_IS_LZMA(method))
return m_lzma;
assert(method == -1); // --all-methods
if (level == 1 || small)
return m_nrv2b;
return m_nrv2e;
@ -1363,16 +1391,20 @@ void Packer::compressWithFilters(Filter *parm_ft,
filters[nfilters] = -1;
// methods
int tmp_methods[] = { ph.method, -1 };
int tmp_methods[] = { ph.method, M_END };
const int *methods = NULL;
if (opt->all_methods)
methods = getCompressionMethods(-1, ph.level);
if (methods == NULL)
methods = tmp_methods;
int nmethods = 0;
while (methods[nmethods] >= 0)
for (int mm = 0; methods[mm] != M_END; ++mm)
{
assert(isValidCompressionMethod(methods[nmethods]));
if (methods[mm] == M_SKIP)
continue;
if (opt->all_methods && !opt->all_methods_use_lzma && methods[mm] == M_LZMA)
continue;
assert(isValidCompressionMethod(methods[mm]));
nmethods++;
}
assert(nmethods > 0);
@ -1389,17 +1421,23 @@ void Packer::compressWithFilters(Filter *parm_ft,
// compress
int nfilters_success = 0;
for (int m = 0; m < nmethods; m++) // for all methods
for (int mm = 0; methods[mm] != M_END; ++mm) // for all methods
{
if (methods[mm] == M_SKIP)
continue;
if (opt->all_methods && !opt->all_methods_use_lzma && methods[mm] == M_LZMA)
continue;
unsigned hdr_c_len = 0;
if (hdr_buf && hdr_u_len)
{
if (0 < m && otemp == &obuf) { // do not overwrite obuf
if (nfilters_success != 0 && otemp == &obuf)
{
// do not overwrite obuf
otemp_buf.allocForCompression(compress_buf_len);
otemp = &otemp_buf;
}
int r = upx_compress(hdr_buf, hdr_u_len, *otemp, &hdr_c_len,
NULL, methods[m], 10, NULL, NULL);
NULL, methods[mm], 10, NULL, NULL);
if (r != UPX_E_OK)
throwInternalError("header compression failed");
if (hdr_c_len >= hdr_u_len)
@ -1411,7 +1449,7 @@ void Packer::compressWithFilters(Filter *parm_ft,
obuf.checkState();
// get fresh packheader
ph = orig_ph;
ph.method = methods[m];
ph.method = methods[mm];
ph.filter = filters[i];
ph.overlap_overhead = 0;
// get fresh filter
@ -1442,7 +1480,7 @@ void Packer::compressWithFilters(Filter *parm_ft,
printf("filter: id 0x%02x size %6d, calls %5d/%5d/%3d/%5d/%5d, cto 0x%02x\n",
ft.id, ft.buf_len, ft.calls, ft.noncalls, ft.wrongcalls, ft.firstcall, ft.lastcall, ft.cto);
#endif
if (nfilters_success > 0 && otemp == &obuf)
if (nfilters_success != 0 && otemp == &obuf)
{
otemp_buf.allocForCompression(compress_buf_len);
otemp = &otemp_buf;

3
src/packer.h
View File

@ -137,6 +137,9 @@ protected:
// unpacker tests - these may throw exceptions
virtual bool testUnpackVersion(int version) const;
virtual bool testUnpackFormat(int format) const;
//
virtual bool skipVerify(int method, int level) const;
virtual bool skipVerify() const;
protected:
// implementation