C/Lzma2Enc.c

/* Lzma2Enc.c -- LZMA2 Encoder
2010-09-24 : Igor Pavlov : Public domain */

/* #include <stdio.h> */
#include <string.h>

/* #define _7ZIP_ST */

#include "Lzma2Enc.h"

#ifndef _7ZIP_ST
#include "MtCoder.h"
#else
#define NUM_MT_CODER_THREADS_MAX 1
#endif

#define LZMA2_CONTROL_LZMA (1 << 7)
#define LZMA2_CONTROL_COPY_NO_RESET 2
#define LZMA2_CONTROL_COPY_RESET_DIC 1
#define LZMA2_CONTROL_EOF 0

#define LZMA2_LCLP_MAX 4

#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))

#define LZMA2_PACK_SIZE_MAX (1 << 16)
#define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX
#define LZMA2_UNPACK_SIZE_MAX (1 << 21)
#define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX

#define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16)


#define PRF(x) /* x */

/* ---------- CLzma2EncInt ---------- */

typedef struct
{
  CLzmaEncHandle enc;
  UInt64 srcPos;
  Byte props;
  Bool needInitState;
  Bool needInitProp;
} CLzma2EncInt;

static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
{
  Byte propsEncoded[LZMA_PROPS_SIZE];
  SizeT propsSize = LZMA_PROPS_SIZE;
  RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
  RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
  p->srcPos = 0;
  p->props = propsEncoded[0];
  p->needInitState = True;
  p->needInitProp = True;
  return SZ_OK;
}

SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
    ISzAlloc *alloc, ISzAlloc *allocBig);
SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
    UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig);
SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
    Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize);
const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp);
void LzmaEnc_Finish(CLzmaEncHandle pp);
void LzmaEnc_SaveState(CLzmaEncHandle pp);
void LzmaEnc_RestoreState(CLzmaEncHandle pp);


static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
    size_t *packSizeRes, ISeqOutStream *outStream)
{
  size_t packSizeLimit = *packSizeRes;
  size_t packSize = packSizeLimit;
  UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX;
  unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0);
  Bool useCopyBlock;
  SRes res;

  *packSizeRes = 0;
  if (packSize < lzHeaderSize)
    return SZ_ERROR_OUTPUT_EOF;
  packSize -= lzHeaderSize;
  
  LzmaEnc_SaveState(p->enc);
  res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState,
      outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize);
  
  PRF(printf("\npackSize = %7d unpackSize = %7d  ", packSize, unpackSize));

  if (unpackSize == 0)
    return res;

  if (res == SZ_OK)
    useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16));
  else
  {
    if (res != SZ_ERROR_OUTPUT_EOF)
      return res;
    res = SZ_OK;
    useCopyBlock = True;
  }

  if (useCopyBlock)
  {
    size_t destPos = 0;
    PRF(printf("################# COPY           "));
    while (unpackSize > 0)
    {
      UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE;
      if (packSizeLimit - destPos < u + 3)
        return SZ_ERROR_OUTPUT_EOF;
      outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET);
      outBuf[destPos++] = (Byte)((u - 1) >> 8);
      outBuf[destPos++] = (Byte)(u - 1);
      memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u);
      unpackSize -= u;
      destPos += u;
      p->srcPos += u;
      if (outStream)
      {
        *packSizeRes += destPos;
        if (outStream->Write(outStream, outBuf, destPos) != destPos)
          return SZ_ERROR_WRITE;
        destPos = 0;
      }
      else
        *packSizeRes = destPos;
      /* needInitState = True; */
    }
    LzmaEnc_RestoreState(p->enc);
    return SZ_OK;
  }
  {
    size_t destPos = 0;
    UInt32 u = unpackSize - 1;
    UInt32 pm = (UInt32)(packSize - 1);
    unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0);

    PRF(printf("               "));

    outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F));
    outBuf[destPos++] = (Byte)(u >> 8);
    outBuf[destPos++] = (Byte)u;
    outBuf[destPos++] = (Byte)(pm >> 8);
    outBuf[destPos++] = (Byte)pm;
    
    if (p->needInitProp)
      outBuf[destPos++] = p->props;
    
    p->needInitProp = False;
    p->needInitState = False;
    destPos += packSize;
    p->srcPos += unpackSize;

    if (outStream)
      if (outStream->Write(outStream, outBuf, destPos) != destPos)
        return SZ_ERROR_WRITE;
    *packSizeRes = destPos;
    return SZ_OK;
  }
}

/* ---------- Lzma2 Props ---------- */

void Lzma2EncProps_Init(CLzma2EncProps *p)
{
  LzmaEncProps_Init(&p->lzmaProps);
  p->numTotalThreads = -1;
  p->numBlockThreads = -1;
  p->blockSize = 0;
}

void Lzma2EncProps_Normalize(CLzma2EncProps *p)
{
  int t1, t1n, t2, t3;
  {
    CLzmaEncProps lzmaProps = p->lzmaProps;
    LzmaEncProps_Normalize(&lzmaProps);
    t1n = lzmaProps.numThreads;
  }

  t1 = p->lzmaProps.numThreads;
  t2 = p->numBlockThreads;
  t3 = p->numTotalThreads;

  if (t2 > NUM_MT_CODER_THREADS_MAX)
    t2 = NUM_MT_CODER_THREADS_MAX;

  if (t3 <= 0)
  {
    if (t2 <= 0)
      t2 = 1;
    t3 = t1n * t2;
  }
  else if (t2 <= 0)
  {
    t2 = t3 / t1n;
    if (t2 == 0)
    {
      t1 = 1;
      t2 = t3;
    }
    if (t2 > NUM_MT_CODER_THREADS_MAX)
      t2 = NUM_MT_CODER_THREADS_MAX;
  }
  else if (t1 <= 0)
  {
    t1 = t3 / t2;
    if (t1 == 0)
      t1 = 1;
  }
  else
    t3 = t1n * t2;

  p->lzmaProps.numThreads = t1;
  p->numBlockThreads = t2;
  p->numTotalThreads = t3;
  LzmaEncProps_Normalize(&p->lzmaProps);

  if (p->blockSize == 0)
  {
    UInt32 dictSize = p->lzmaProps.dictSize;
    UInt64 blockSize = (UInt64)dictSize << 2;
    const UInt32 kMinSize = (UInt32)1 << 20;
    const UInt32 kMaxSize = (UInt32)1 << 28;
    if (blockSize < kMinSize) blockSize = kMinSize;
    if (blockSize > kMaxSize) blockSize = kMaxSize;
    if (blockSize < dictSize) blockSize = dictSize;
    p->blockSize = (size_t)blockSize;
  }
}

static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
{
  return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
}

/* ---------- Lzma2 ---------- */

typedef struct
{
  Byte propEncoded;
  CLzma2EncProps props;
  
  Byte *outBuf;

  ISzAlloc *alloc;
  ISzAlloc *allocBig;

  CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];

  #ifndef _7ZIP_ST
  CMtCoder mtCoder;
  #endif

} CLzma2Enc;


/* ---------- Lzma2EncThread ---------- */

static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
  ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
{
  UInt64 packTotal = 0;
  SRes res = SZ_OK;

  if (mainEncoder->outBuf == 0)
  {
    mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
    if (mainEncoder->outBuf == 0)
      return SZ_ERROR_MEM;
  }
  RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
  RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
      mainEncoder->alloc, mainEncoder->allocBig));
  for (;;)
  {
    size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
    res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
    if (res != SZ_OK)
      break;
    packTotal += packSize;
    res = Progress(progress, p->srcPos, packTotal);
    if (res != SZ_OK)
      break;
    if (packSize == 0)
      break;
  }
  LzmaEnc_Finish(p->enc);
  if (res == SZ_OK)
  {
    Byte b = 0;
    if (outStream->Write(outStream, &b, 1) != 1)
      return SZ_ERROR_WRITE;
  }
  return res;
}

#ifndef _7ZIP_ST

typedef struct
{
  IMtCoderCallback funcTable;
  CLzma2Enc *lzma2Enc;
} CMtCallbackImp;

static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize,
      const Byte *src, size_t srcSize, int finished)
{
  CMtCallbackImp *imp = (CMtCallbackImp *)pp;
  CLzma2Enc *mainEncoder = imp->lzma2Enc;
  CLzma2EncInt *p = &mainEncoder->coders[index];

  SRes res = SZ_OK;
  {
    size_t destLim = *destSize;
    *destSize = 0;

    if (srcSize != 0)
    {
      RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
     
      RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
          mainEncoder->alloc, mainEncoder->allocBig));
     
      while (p->srcPos < srcSize)
      {
        size_t packSize = destLim - *destSize;
        res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
        if (res != SZ_OK)
          break;
        *destSize += packSize;

        if (packSize == 0)
        {
          res = SZ_ERROR_FAIL;
          break;
        }

        if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
        {
          res = SZ_ERROR_PROGRESS;
          break;
        }
      }
      LzmaEnc_Finish(p->enc);
      if (res != SZ_OK)
        return res;
    }
    if (finished)
    {
      if (*destSize == destLim)
        return SZ_ERROR_OUTPUT_EOF;
      dest[(*destSize)++] = 0;
    }
  }
  return res;
}

#endif

/* ---------- Lzma2Enc ---------- */

CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig)
{
  CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc));
  if (p == 0)
    return NULL;
  Lzma2EncProps_Init(&p->props);
  Lzma2EncProps_Normalize(&p->props);
  p->outBuf = 0;
  p->alloc = alloc;
  p->allocBig = allocBig;
  {
    unsigned i;
    for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
      p->coders[i].enc = 0;
  }
  #ifndef _7ZIP_ST
  MtCoder_Construct(&p->mtCoder);
  #endif

  return p;
}

void Lzma2Enc_Destroy(CLzma2EncHandle pp)
{
  CLzma2Enc *p = (CLzma2Enc *)pp;
  unsigned i;
  for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
  {
    CLzma2EncInt *t = &p->coders[i];
    if (t->enc)
    {
      LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
      t->enc = 0;
    }
  }

  #ifndef _7ZIP_ST
  MtCoder_Destruct(&p->mtCoder);
  #endif

  IAlloc_Free(p->alloc, p->outBuf);
  IAlloc_Free(p->alloc, pp);
}

SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
{
  CLzma2Enc *p = (CLzma2Enc *)pp;
  CLzmaEncProps lzmaProps = props->lzmaProps;
  LzmaEncProps_Normalize(&lzmaProps);
  if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX)
    return SZ_ERROR_PARAM;
  p->props = *props;
  Lzma2EncProps_Normalize(&p->props);
  return SZ_OK;
}

Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
{
  CLzma2Enc *p = (CLzma2Enc *)pp;
  unsigned i;
  UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps);
  for (i = 0; i < 40; i++)
    if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
      break;
  return (Byte)i;
}

SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
    ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
{
  CLzma2Enc *p = (CLzma2Enc *)pp;
  int i;

  for (i = 0; i < p->props.numBlockThreads; i++)
  {
    CLzma2EncInt *t = &p->coders[i];
    if (t->enc == NULL)
    {
      t->enc = LzmaEnc_Create(p->alloc);
      if (t->enc == NULL)
        return SZ_ERROR_MEM;
    }
  }

  #ifndef _7ZIP_ST
  if (p->props.numBlockThreads <= 1)
  #endif
    return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);

  #ifndef _7ZIP_ST

  {
    CMtCallbackImp mtCallback;

    mtCallback.funcTable.Code = MtCallbackImp_Code;
    mtCallback.lzma2Enc = p;
    
    p->mtCoder.progress = progress;
    p->mtCoder.inStream = inStream;
    p->mtCoder.outStream = outStream;
    p->mtCoder.alloc = p->alloc;
    p->mtCoder.mtCallback = &mtCallback.funcTable;

    p->mtCoder.blockSize = p->props.blockSize;
    p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
    p->mtCoder.numThreads = p->props.numBlockThreads;
    
    return MtCoder_Code(&p->mtCoder);
  }
  #endif
}