blob: 7b501d8202d8b99e927f42708ae86349dc279b0d [file] [log] [blame]
Larry Hastingsf256c222014-01-25 21:30:37 -08001/* _lzma - Low-level Python interface to liblzma.
2
3 Initial implementation by Per Øyvind Karlsen.
4 Rewritten by Nadeem Vawda.
5
6*/
7
8#define PY_SSIZE_T_CLEAN
9
10#include "Python.h"
11#include "structmember.h"
Larry Hastingsf256c222014-01-25 21:30:37 -080012#include "pythread.h"
Larry Hastingsf256c222014-01-25 21:30:37 -080013
14#include <stdarg.h>
15#include <string.h>
16
17#include <lzma.h>
18
Larry Hastingsf256c222014-01-25 21:30:37 -080019#define ACQUIRE_LOCK(obj) do { \
20 if (!PyThread_acquire_lock((obj)->lock, 0)) { \
21 Py_BEGIN_ALLOW_THREADS \
22 PyThread_acquire_lock((obj)->lock, 1); \
23 Py_END_ALLOW_THREADS \
24 } } while (0)
25#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock)
Larry Hastingsf256c222014-01-25 21:30:37 -080026
27
28/* Container formats: */
29enum {
30 FORMAT_AUTO,
31 FORMAT_XZ,
32 FORMAT_ALONE,
33 FORMAT_RAW,
34};
35
36#define LZMA_CHECK_UNKNOWN (LZMA_CHECK_ID_MAX + 1)
37
38
39typedef struct {
40 PyObject_HEAD
41 lzma_allocator alloc;
42 lzma_stream lzs;
43 int flushed;
Larry Hastingsf256c222014-01-25 21:30:37 -080044 PyThread_type_lock lock;
Larry Hastingsf256c222014-01-25 21:30:37 -080045} Compressor;
46
47typedef struct {
48 PyObject_HEAD
49 lzma_allocator alloc;
50 lzma_stream lzs;
51 int check;
52 char eof;
53 PyObject *unused_data;
Antoine Pitrou26795ba2015-01-17 16:22:18 +010054 char needs_input;
55 uint8_t *input_buffer;
56 size_t input_buffer_size;
Larry Hastingsf256c222014-01-25 21:30:37 -080057 PyThread_type_lock lock;
Larry Hastingsf256c222014-01-25 21:30:37 -080058} Decompressor;
59
60/* LZMAError class object. */
61static PyObject *Error;
62
63/* An empty tuple, used by the filter specifier parsing code. */
64static PyObject *empty_tuple;
65
66
67/* Helper functions. */
68
69static int
70catch_lzma_error(lzma_ret lzret)
71{
72 switch (lzret) {
73 case LZMA_OK:
74 case LZMA_GET_CHECK:
75 case LZMA_NO_CHECK:
76 case LZMA_STREAM_END:
77 return 0;
78 case LZMA_UNSUPPORTED_CHECK:
79 PyErr_SetString(Error, "Unsupported integrity check");
80 return 1;
81 case LZMA_MEM_ERROR:
82 PyErr_NoMemory();
83 return 1;
84 case LZMA_MEMLIMIT_ERROR:
85 PyErr_SetString(Error, "Memory usage limit exceeded");
86 return 1;
87 case LZMA_FORMAT_ERROR:
88 PyErr_SetString(Error, "Input format not supported by decoder");
89 return 1;
90 case LZMA_OPTIONS_ERROR:
91 PyErr_SetString(Error, "Invalid or unsupported options");
92 return 1;
93 case LZMA_DATA_ERROR:
94 PyErr_SetString(Error, "Corrupt input data");
95 return 1;
96 case LZMA_BUF_ERROR:
97 PyErr_SetString(Error, "Insufficient buffer space");
98 return 1;
99 case LZMA_PROG_ERROR:
100 PyErr_SetString(Error, "Internal error");
101 return 1;
102 default:
103 PyErr_Format(Error, "Unrecognized error from liblzma: %d", lzret);
104 return 1;
105 }
106}
107
108static void*
109PyLzma_Malloc(void *opaque, size_t items, size_t size)
110{
111 if (items > (size_t)PY_SSIZE_T_MAX / size)
112 return NULL;
113 /* PyMem_Malloc() cannot be used:
114 the GIL is not held when lzma_code() is called */
115 return PyMem_RawMalloc(items * size);
116}
117
118static void
119PyLzma_Free(void *opaque, void *ptr)
120{
121 PyMem_RawFree(ptr);
122}
123
124#if BUFSIZ < 8192
125#define INITIAL_BUFFER_SIZE 8192
126#else
127#define INITIAL_BUFFER_SIZE BUFSIZ
128#endif
129
130static int
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100131grow_buffer(PyObject **buf, Py_ssize_t max_length)
Larry Hastingsf256c222014-01-25 21:30:37 -0800132{
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100133 Py_ssize_t size = PyBytes_GET_SIZE(*buf);
134 Py_ssize_t newsize = size + (size >> 3) + 6;
Serhiy Storchaka009b8112015-03-18 21:53:15 +0200135
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100136 if (max_length > 0 && newsize > max_length)
137 newsize = max_length;
Serhiy Storchaka009b8112015-03-18 21:53:15 +0200138
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100139 return _PyBytes_Resize(buf, newsize);
Larry Hastingsf256c222014-01-25 21:30:37 -0800140}
141
142
143/* Some custom type conversions for PyArg_ParseTupleAndKeywords(),
144 since the predefined conversion specifiers do not suit our needs:
145
146 uint32_t - the "I" (unsigned int) specifier is the right size, but
147 silently ignores overflows on conversion.
148
Benjamin Petersonaf580df2016-09-06 10:46:49 -0700149 lzma_vli - the "K" (unsigned long long) specifier is the right
Larry Hastingsf256c222014-01-25 21:30:37 -0800150 size, but like "I" it silently ignores overflows on conversion.
151
152 lzma_mode and lzma_match_finder - these are enumeration types, and
153 so the size of each is implementation-defined. Worse, different
154 enum types can be of different sizes within the same program, so
155 to be strictly correct, we need to define two separate converters.
156 */
157
158#define INT_TYPE_CONVERTER_FUNC(TYPE, FUNCNAME) \
159 static int \
160 FUNCNAME(PyObject *obj, void *ptr) \
161 { \
Benjamin Petersonaf580df2016-09-06 10:46:49 -0700162 unsigned long long val; \
Larry Hastingsf256c222014-01-25 21:30:37 -0800163 \
164 val = PyLong_AsUnsignedLongLong(obj); \
165 if (PyErr_Occurred()) \
166 return 0; \
Benjamin Petersonaf580df2016-09-06 10:46:49 -0700167 if ((unsigned long long)(TYPE)val != val) { \
Larry Hastingsf256c222014-01-25 21:30:37 -0800168 PyErr_SetString(PyExc_OverflowError, \
169 "Value too large for " #TYPE " type"); \
170 return 0; \
171 } \
172 *(TYPE *)ptr = (TYPE)val; \
173 return 1; \
174 }
175
176INT_TYPE_CONVERTER_FUNC(uint32_t, uint32_converter)
177INT_TYPE_CONVERTER_FUNC(lzma_vli, lzma_vli_converter)
178INT_TYPE_CONVERTER_FUNC(lzma_mode, lzma_mode_converter)
179INT_TYPE_CONVERTER_FUNC(lzma_match_finder, lzma_mf_converter)
180
181#undef INT_TYPE_CONVERTER_FUNC
182
183
184/* Filter specifier parsing.
185
186 This code handles converting filter specifiers (Python dicts) into
187 the C lzma_filter structs expected by liblzma. */
188
189static void *
190parse_filter_spec_lzma(PyObject *spec)
191{
192 static char *optnames[] = {"id", "preset", "dict_size", "lc", "lp",
193 "pb", "mode", "nice_len", "mf", "depth", NULL};
194 PyObject *id;
195 PyObject *preset_obj;
196 uint32_t preset = LZMA_PRESET_DEFAULT;
197 lzma_options_lzma *options;
198
199 /* First, fill in default values for all the options using a preset.
200 Then, override the defaults with any values given by the caller. */
201
202 preset_obj = PyMapping_GetItemString(spec, "preset");
203 if (preset_obj == NULL) {
204 if (PyErr_ExceptionMatches(PyExc_KeyError))
205 PyErr_Clear();
206 else
207 return NULL;
208 } else {
209 int ok = uint32_converter(preset_obj, &preset);
210 Py_DECREF(preset_obj);
211 if (!ok)
212 return NULL;
213 }
214
215 options = (lzma_options_lzma *)PyMem_Malloc(sizeof *options);
216 if (options == NULL)
217 return PyErr_NoMemory();
218 memset(options, 0, sizeof *options);
219
220 if (lzma_lzma_preset(options, preset)) {
221 PyMem_Free(options);
222 PyErr_Format(Error, "Invalid compression preset: %d", preset);
223 return NULL;
224 }
225
226 if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec,
227 "|OOO&O&O&O&O&O&O&O&", optnames,
228 &id, &preset_obj,
229 uint32_converter, &options->dict_size,
230 uint32_converter, &options->lc,
231 uint32_converter, &options->lp,
232 uint32_converter, &options->pb,
233 lzma_mode_converter, &options->mode,
234 uint32_converter, &options->nice_len,
235 lzma_mf_converter, &options->mf,
236 uint32_converter, &options->depth)) {
237 PyErr_SetString(PyExc_ValueError,
238 "Invalid filter specifier for LZMA filter");
239 PyMem_Free(options);
240 options = NULL;
241 }
242 return options;
243}
244
245static void *
246parse_filter_spec_delta(PyObject *spec)
247{
248 static char *optnames[] = {"id", "dist", NULL};
249 PyObject *id;
250 uint32_t dist = 1;
251 lzma_options_delta *options;
252
253 if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec, "|OO&", optnames,
254 &id, uint32_converter, &dist)) {
255 PyErr_SetString(PyExc_ValueError,
256 "Invalid filter specifier for delta filter");
257 return NULL;
258 }
259
260 options = (lzma_options_delta *)PyMem_Malloc(sizeof *options);
261 if (options == NULL)
262 return PyErr_NoMemory();
263 memset(options, 0, sizeof *options);
264 options->type = LZMA_DELTA_TYPE_BYTE;
265 options->dist = dist;
266 return options;
267}
268
269static void *
270parse_filter_spec_bcj(PyObject *spec)
271{
272 static char *optnames[] = {"id", "start_offset", NULL};
273 PyObject *id;
274 uint32_t start_offset = 0;
275 lzma_options_bcj *options;
276
277 if (!PyArg_ParseTupleAndKeywords(empty_tuple, spec, "|OO&", optnames,
278 &id, uint32_converter, &start_offset)) {
279 PyErr_SetString(PyExc_ValueError,
280 "Invalid filter specifier for BCJ filter");
281 return NULL;
282 }
283
284 options = (lzma_options_bcj *)PyMem_Malloc(sizeof *options);
285 if (options == NULL)
286 return PyErr_NoMemory();
287 memset(options, 0, sizeof *options);
288 options->start_offset = start_offset;
289 return options;
290}
291
292static int
293lzma_filter_converter(PyObject *spec, void *ptr)
294{
295 lzma_filter *f = (lzma_filter *)ptr;
296 PyObject *id_obj;
297
298 if (!PyMapping_Check(spec)) {
299 PyErr_SetString(PyExc_TypeError,
300 "Filter specifier must be a dict or dict-like object");
301 return 0;
302 }
303 id_obj = PyMapping_GetItemString(spec, "id");
304 if (id_obj == NULL) {
305 if (PyErr_ExceptionMatches(PyExc_KeyError))
306 PyErr_SetString(PyExc_ValueError,
307 "Filter specifier must have an \"id\" entry");
308 return 0;
309 }
310 f->id = PyLong_AsUnsignedLongLong(id_obj);
311 Py_DECREF(id_obj);
312 if (PyErr_Occurred())
313 return 0;
314
315 switch (f->id) {
316 case LZMA_FILTER_LZMA1:
317 case LZMA_FILTER_LZMA2:
318 f->options = parse_filter_spec_lzma(spec);
319 return f->options != NULL;
320 case LZMA_FILTER_DELTA:
321 f->options = parse_filter_spec_delta(spec);
322 return f->options != NULL;
323 case LZMA_FILTER_X86:
324 case LZMA_FILTER_POWERPC:
325 case LZMA_FILTER_IA64:
326 case LZMA_FILTER_ARM:
327 case LZMA_FILTER_ARMTHUMB:
328 case LZMA_FILTER_SPARC:
329 f->options = parse_filter_spec_bcj(spec);
330 return f->options != NULL;
331 default:
332 PyErr_Format(PyExc_ValueError, "Invalid filter ID: %llu", f->id);
333 return 0;
334 }
335}
336
337static void
338free_filter_chain(lzma_filter filters[])
339{
340 int i;
341
342 for (i = 0; filters[i].id != LZMA_VLI_UNKNOWN; i++)
343 PyMem_Free(filters[i].options);
344}
345
346static int
347parse_filter_chain_spec(lzma_filter filters[], PyObject *filterspecs)
348{
349 Py_ssize_t i, num_filters;
350
351 num_filters = PySequence_Length(filterspecs);
352 if (num_filters == -1)
353 return -1;
354 if (num_filters > LZMA_FILTERS_MAX) {
355 PyErr_Format(PyExc_ValueError,
356 "Too many filters - liblzma supports a maximum of %d",
357 LZMA_FILTERS_MAX);
358 return -1;
359 }
360
361 for (i = 0; i < num_filters; i++) {
362 int ok = 1;
363 PyObject *spec = PySequence_GetItem(filterspecs, i);
364 if (spec == NULL || !lzma_filter_converter(spec, &filters[i]))
365 ok = 0;
366 Py_XDECREF(spec);
367 if (!ok) {
368 filters[i].id = LZMA_VLI_UNKNOWN;
369 free_filter_chain(filters);
370 return -1;
371 }
372 }
373 filters[num_filters].id = LZMA_VLI_UNKNOWN;
374 return 0;
375}
376
377
378/* Filter specifier construction.
379
380 This code handles converting C lzma_filter structs into
381 Python-level filter specifiers (represented as dicts). */
382
383static int
Benjamin Petersonaf580df2016-09-06 10:46:49 -0700384spec_add_field(PyObject *spec, _Py_Identifier *key, unsigned long long value)
Larry Hastingsf256c222014-01-25 21:30:37 -0800385{
386 int status;
387 PyObject *value_object;
388
389 value_object = PyLong_FromUnsignedLongLong(value);
390 if (value_object == NULL)
391 return -1;
392
393 status = _PyDict_SetItemId(spec, key, value_object);
394 Py_DECREF(value_object);
395 return status;
396}
397
398static PyObject *
399build_filter_spec(const lzma_filter *f)
400{
401 PyObject *spec;
402
403 spec = PyDict_New();
404 if (spec == NULL)
405 return NULL;
406
407#define ADD_FIELD(SOURCE, FIELD) \
408 do { \
409 _Py_IDENTIFIER(FIELD); \
410 if (spec_add_field(spec, &PyId_##FIELD, SOURCE->FIELD) == -1) \
411 goto error;\
412 } while (0)
413
414 ADD_FIELD(f, id);
415
416 switch (f->id) {
417 /* For LZMA1 filters, lzma_properties_{encode,decode}() only look at the
418 lc, lp, pb, and dict_size fields. For LZMA2 filters, only the
419 dict_size field is used. */
420 case LZMA_FILTER_LZMA1: {
421 lzma_options_lzma *options = f->options;
422 ADD_FIELD(options, lc);
423 ADD_FIELD(options, lp);
424 ADD_FIELD(options, pb);
425 ADD_FIELD(options, dict_size);
426 break;
427 }
428 case LZMA_FILTER_LZMA2: {
429 lzma_options_lzma *options = f->options;
430 ADD_FIELD(options, dict_size);
431 break;
432 }
433 case LZMA_FILTER_DELTA: {
434 lzma_options_delta *options = f->options;
435 ADD_FIELD(options, dist);
436 break;
437 }
438 case LZMA_FILTER_X86:
439 case LZMA_FILTER_POWERPC:
440 case LZMA_FILTER_IA64:
441 case LZMA_FILTER_ARM:
442 case LZMA_FILTER_ARMTHUMB:
443 case LZMA_FILTER_SPARC: {
444 lzma_options_bcj *options = f->options;
445 ADD_FIELD(options, start_offset);
446 break;
447 }
448 default:
449 PyErr_Format(PyExc_ValueError, "Invalid filter ID: %llu", f->id);
450 goto error;
451 }
452
453#undef ADD_FIELD
454
455 return spec;
456
457error:
458 Py_DECREF(spec);
459 return NULL;
460}
461
462
463/*[clinic input]
Larry Hastingsf256c222014-01-25 21:30:37 -0800464module _lzma
465class _lzma.LZMACompressor "Compressor *" "&Compressor_type"
466class _lzma.LZMADecompressor "Decompressor *" "&Decompressor_type"
467[clinic start generated code]*/
Serhiy Storchaka1009bf12015-04-03 23:53:51 +0300468/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2c14bbe05ff0c147]*/
Larry Hastingsf256c222014-01-25 21:30:37 -0800469
470#include "clinic/_lzmamodule.c.h"
471
472/*[python input]
473
474class lzma_vli_converter(CConverter):
475 type = 'lzma_vli'
476 converter = 'lzma_vli_converter'
477
478class lzma_filter_converter(CConverter):
479 type = 'lzma_filter'
480 converter = 'lzma_filter_converter'
481 c_default = c_ignored_default = "{LZMA_VLI_UNKNOWN, NULL}"
482
483 def cleanup(self):
484 name = ensure_legal_c_identifier(self.name)
485 return ('if (%(name)s.id != LZMA_VLI_UNKNOWN)\n'
486 ' PyMem_Free(%(name)s.options);\n') % {'name': name}
487
488[python start generated code]*/
Larry Hastings581ee362014-01-28 05:00:08 -0800489/*[python end generated code: output=da39a3ee5e6b4b0d input=74fe7631ce377a94]*/
Larry Hastingsf256c222014-01-25 21:30:37 -0800490
491
492/* LZMACompressor class. */
493
494static PyObject *
495compress(Compressor *c, uint8_t *data, size_t len, lzma_action action)
496{
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100497 Py_ssize_t data_size = 0;
Larry Hastingsf256c222014-01-25 21:30:37 -0800498 PyObject *result;
499
500 result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE);
501 if (result == NULL)
502 return NULL;
503 c->lzs.next_in = data;
504 c->lzs.avail_in = len;
505 c->lzs.next_out = (uint8_t *)PyBytes_AS_STRING(result);
506 c->lzs.avail_out = PyBytes_GET_SIZE(result);
507 for (;;) {
508 lzma_ret lzret;
509
510 Py_BEGIN_ALLOW_THREADS
511 lzret = lzma_code(&c->lzs, action);
512 data_size = (char *)c->lzs.next_out - PyBytes_AS_STRING(result);
Serhiy Storchaka04f17f12016-10-31 08:30:09 +0200513 if (lzret == LZMA_BUF_ERROR && len == 0 && c->lzs.avail_out > 0)
514 lzret = LZMA_OK; /* That wasn't a real error */
Larry Hastingsf256c222014-01-25 21:30:37 -0800515 Py_END_ALLOW_THREADS
516 if (catch_lzma_error(lzret))
517 goto error;
518 if ((action == LZMA_RUN && c->lzs.avail_in == 0) ||
519 (action == LZMA_FINISH && lzret == LZMA_STREAM_END)) {
520 break;
521 } else if (c->lzs.avail_out == 0) {
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100522 if (grow_buffer(&result, -1) == -1)
Larry Hastingsf256c222014-01-25 21:30:37 -0800523 goto error;
524 c->lzs.next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size;
525 c->lzs.avail_out = PyBytes_GET_SIZE(result) - data_size;
526 }
527 }
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100528 if (data_size != PyBytes_GET_SIZE(result))
Larry Hastingsf256c222014-01-25 21:30:37 -0800529 if (_PyBytes_Resize(&result, data_size) == -1)
530 goto error;
531 return result;
532
533error:
534 Py_XDECREF(result);
535 return NULL;
536}
537
538/*[clinic input]
539_lzma.LZMACompressor.compress
540
Larry Hastingsf256c222014-01-25 21:30:37 -0800541 data: Py_buffer
542 /
543
544Provide data to the compressor object.
545
546Returns a chunk of compressed data if possible, or b'' otherwise.
547
548When you have finished providing data to the compressor, call the
549flush() method to finish the compression process.
550[clinic start generated code]*/
551
552static PyObject *
553_lzma_LZMACompressor_compress_impl(Compressor *self, Py_buffer *data)
Serhiy Storchaka7a9579c2016-05-02 13:45:20 +0300554/*[clinic end generated code: output=31f615136963e00f input=64019eac7f2cc8d0]*/
Larry Hastingsf256c222014-01-25 21:30:37 -0800555{
556 PyObject *result = NULL;
557
558 ACQUIRE_LOCK(self);
559 if (self->flushed)
560 PyErr_SetString(PyExc_ValueError, "Compressor has been flushed");
561 else
562 result = compress(self, data->buf, data->len, LZMA_RUN);
563 RELEASE_LOCK(self);
564 return result;
565}
566
567/*[clinic input]
568_lzma.LZMACompressor.flush
569
Larry Hastingsf256c222014-01-25 21:30:37 -0800570Finish the compression process.
571
572Returns the compressed data left in internal buffers.
573
574The compressor object may not be used after this method is called.
575[clinic start generated code]*/
576
577static PyObject *
578_lzma_LZMACompressor_flush_impl(Compressor *self)
Serhiy Storchaka7a9579c2016-05-02 13:45:20 +0300579/*[clinic end generated code: output=fec21f3e22504f50 input=6b369303f67ad0a8]*/
Larry Hastingsf256c222014-01-25 21:30:37 -0800580{
581 PyObject *result = NULL;
582
583 ACQUIRE_LOCK(self);
584 if (self->flushed) {
585 PyErr_SetString(PyExc_ValueError, "Repeated call to flush()");
586 } else {
587 self->flushed = 1;
588 result = compress(self, NULL, 0, LZMA_FINISH);
589 }
590 RELEASE_LOCK(self);
591 return result;
592}
593
594static PyObject *
595Compressor_getstate(Compressor *self, PyObject *noargs)
596{
597 PyErr_Format(PyExc_TypeError, "cannot serialize '%s' object",
598 Py_TYPE(self)->tp_name);
599 return NULL;
600}
601
602static int
603Compressor_init_xz(lzma_stream *lzs, int check, uint32_t preset,
604 PyObject *filterspecs)
605{
606 lzma_ret lzret;
607
608 if (filterspecs == Py_None) {
609 lzret = lzma_easy_encoder(lzs, preset, check);
610 } else {
611 lzma_filter filters[LZMA_FILTERS_MAX + 1];
612
613 if (parse_filter_chain_spec(filters, filterspecs) == -1)
614 return -1;
615 lzret = lzma_stream_encoder(lzs, filters, check);
616 free_filter_chain(filters);
617 }
618 if (catch_lzma_error(lzret))
619 return -1;
620 else
621 return 0;
622}
623
624static int
625Compressor_init_alone(lzma_stream *lzs, uint32_t preset, PyObject *filterspecs)
626{
627 lzma_ret lzret;
628
629 if (filterspecs == Py_None) {
630 lzma_options_lzma options;
631
632 if (lzma_lzma_preset(&options, preset)) {
633 PyErr_Format(Error, "Invalid compression preset: %d", preset);
634 return -1;
635 }
636 lzret = lzma_alone_encoder(lzs, &options);
637 } else {
638 lzma_filter filters[LZMA_FILTERS_MAX + 1];
639
640 if (parse_filter_chain_spec(filters, filterspecs) == -1)
641 return -1;
642 if (filters[0].id == LZMA_FILTER_LZMA1 &&
643 filters[1].id == LZMA_VLI_UNKNOWN) {
644 lzret = lzma_alone_encoder(lzs, filters[0].options);
645 } else {
646 PyErr_SetString(PyExc_ValueError,
647 "Invalid filter chain for FORMAT_ALONE - "
648 "must be a single LZMA1 filter");
649 lzret = LZMA_PROG_ERROR;
650 }
651 free_filter_chain(filters);
652 }
653 if (PyErr_Occurred() || catch_lzma_error(lzret))
654 return -1;
655 else
656 return 0;
657}
658
659static int
660Compressor_init_raw(lzma_stream *lzs, PyObject *filterspecs)
661{
662 lzma_filter filters[LZMA_FILTERS_MAX + 1];
663 lzma_ret lzret;
664
665 if (filterspecs == Py_None) {
666 PyErr_SetString(PyExc_ValueError,
667 "Must specify filters for FORMAT_RAW");
668 return -1;
669 }
670 if (parse_filter_chain_spec(filters, filterspecs) == -1)
671 return -1;
672 lzret = lzma_raw_encoder(lzs, filters);
673 free_filter_chain(filters);
674 if (catch_lzma_error(lzret))
675 return -1;
676 else
677 return 0;
678}
679
680/*[-clinic input]
681_lzma.LZMACompressor.__init__
682
Larry Hastingsf256c222014-01-25 21:30:37 -0800683 format: int(c_default="FORMAT_XZ") = FORMAT_XZ
684 The container format to use for the output. This can
685 be FORMAT_XZ (default), FORMAT_ALONE, or FORMAT_RAW.
686
687 check: int(c_default="-1") = unspecified
688 The integrity check to use. For FORMAT_XZ, the default
Martin Pantere26da7c2016-06-02 10:07:09 +0000689 is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not support integrity
Larry Hastingsf256c222014-01-25 21:30:37 -0800690 checks; for these formats, check must be omitted, or be CHECK_NONE.
691
692 preset: object = None
693 If provided should be an integer in the range 0-9, optionally
694 OR-ed with the constant PRESET_EXTREME.
695
696 filters: object = None
697 If provided should be a sequence of dicts. Each dict should
698 have an entry for "id" indicating the ID of the filter, plus
699 additional entries for options to the filter.
700
701Create a compressor object for compressing data incrementally.
702
703The settings used by the compressor can be specified either as a
704preset compression level (with the 'preset' argument), or in detail
705as a custom filter chain (with the 'filters' argument). For FORMAT_XZ
706and FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset
707level. For FORMAT_RAW, the caller must always specify a filter chain;
708the raw compressor does not support preset compression levels.
709
710For one-shot compression, use the compress() function instead.
711[-clinic start generated code]*/
712static int
713Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs)
714{
715 static char *arg_names[] = {"format", "check", "preset", "filters", NULL};
716 int format = FORMAT_XZ;
717 int check = -1;
718 uint32_t preset = LZMA_PRESET_DEFAULT;
719 PyObject *preset_obj = Py_None;
720 PyObject *filterspecs = Py_None;
721
722 if (!PyArg_ParseTupleAndKeywords(args, kwargs,
723 "|iiOO:LZMACompressor", arg_names,
724 &format, &check, &preset_obj,
725 &filterspecs))
726 return -1;
727
728 if (format != FORMAT_XZ && check != -1 && check != LZMA_CHECK_NONE) {
729 PyErr_SetString(PyExc_ValueError,
730 "Integrity checks are only supported by FORMAT_XZ");
731 return -1;
732 }
733
734 if (preset_obj != Py_None && filterspecs != Py_None) {
735 PyErr_SetString(PyExc_ValueError,
736 "Cannot specify both preset and filter chain");
737 return -1;
738 }
739
740 if (preset_obj != Py_None)
741 if (!uint32_converter(preset_obj, &preset))
742 return -1;
743
744 self->alloc.opaque = NULL;
745 self->alloc.alloc = PyLzma_Malloc;
746 self->alloc.free = PyLzma_Free;
747 self->lzs.allocator = &self->alloc;
748
Larry Hastingsf256c222014-01-25 21:30:37 -0800749 self->lock = PyThread_allocate_lock();
750 if (self->lock == NULL) {
751 PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock");
752 return -1;
753 }
Larry Hastingsf256c222014-01-25 21:30:37 -0800754
755 self->flushed = 0;
756 switch (format) {
757 case FORMAT_XZ:
758 if (check == -1)
759 check = LZMA_CHECK_CRC64;
760 if (Compressor_init_xz(&self->lzs, check, preset, filterspecs) != 0)
761 break;
762 return 0;
763
764 case FORMAT_ALONE:
765 if (Compressor_init_alone(&self->lzs, preset, filterspecs) != 0)
766 break;
767 return 0;
768
769 case FORMAT_RAW:
770 if (Compressor_init_raw(&self->lzs, filterspecs) != 0)
771 break;
772 return 0;
773
774 default:
775 PyErr_Format(PyExc_ValueError,
776 "Invalid container format: %d", format);
777 break;
778 }
779
Larry Hastingsf256c222014-01-25 21:30:37 -0800780 PyThread_free_lock(self->lock);
781 self->lock = NULL;
Larry Hastingsf256c222014-01-25 21:30:37 -0800782 return -1;
783}
784
785static void
786Compressor_dealloc(Compressor *self)
787{
788 lzma_end(&self->lzs);
Larry Hastingsf256c222014-01-25 21:30:37 -0800789 if (self->lock != NULL)
790 PyThread_free_lock(self->lock);
Larry Hastingsf256c222014-01-25 21:30:37 -0800791 Py_TYPE(self)->tp_free((PyObject *)self);
792}
793
794static PyMethodDef Compressor_methods[] = {
795 _LZMA_LZMACOMPRESSOR_COMPRESS_METHODDEF
796 _LZMA_LZMACOMPRESSOR_FLUSH_METHODDEF
797 {"__getstate__", (PyCFunction)Compressor_getstate, METH_NOARGS},
798 {NULL}
799};
800
801PyDoc_STRVAR(Compressor_doc,
802"LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)\n"
803"\n"
804"Create a compressor object for compressing data incrementally.\n"
805"\n"
806"format specifies the container format to use for the output. This can\n"
807"be FORMAT_XZ (default), FORMAT_ALONE, or FORMAT_RAW.\n"
808"\n"
809"check specifies the integrity check to use. For FORMAT_XZ, the default\n"
Ville Skyttä49b27342017-08-03 09:00:59 +0300810"is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not support integrity\n"
Larry Hastingsf256c222014-01-25 21:30:37 -0800811"checks; for these formats, check must be omitted, or be CHECK_NONE.\n"
812"\n"
813"The settings used by the compressor can be specified either as a\n"
814"preset compression level (with the 'preset' argument), or in detail\n"
815"as a custom filter chain (with the 'filters' argument). For FORMAT_XZ\n"
816"and FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset\n"
817"level. For FORMAT_RAW, the caller must always specify a filter chain;\n"
818"the raw compressor does not support preset compression levels.\n"
819"\n"
820"preset (if provided) should be an integer in the range 0-9, optionally\n"
821"OR-ed with the constant PRESET_EXTREME.\n"
822"\n"
823"filters (if provided) should be a sequence of dicts. Each dict should\n"
824"have an entry for \"id\" indicating the ID of the filter, plus\n"
825"additional entries for options to the filter.\n"
826"\n"
827"For one-shot compression, use the compress() function instead.\n");
828
829static PyTypeObject Compressor_type = {
830 PyVarObject_HEAD_INIT(NULL, 0)
831 "_lzma.LZMACompressor", /* tp_name */
832 sizeof(Compressor), /* tp_basicsize */
833 0, /* tp_itemsize */
834 (destructor)Compressor_dealloc, /* tp_dealloc */
835 0, /* tp_print */
836 0, /* tp_getattr */
837 0, /* tp_setattr */
838 0, /* tp_reserved */
839 0, /* tp_repr */
840 0, /* tp_as_number */
841 0, /* tp_as_sequence */
842 0, /* tp_as_mapping */
843 0, /* tp_hash */
844 0, /* tp_call */
845 0, /* tp_str */
846 0, /* tp_getattro */
847 0, /* tp_setattro */
848 0, /* tp_as_buffer */
849 Py_TPFLAGS_DEFAULT, /* tp_flags */
850 Compressor_doc, /* tp_doc */
851 0, /* tp_traverse */
852 0, /* tp_clear */
853 0, /* tp_richcompare */
854 0, /* tp_weaklistoffset */
855 0, /* tp_iter */
856 0, /* tp_iternext */
857 Compressor_methods, /* tp_methods */
858 0, /* tp_members */
859 0, /* tp_getset */
860 0, /* tp_base */
861 0, /* tp_dict */
862 0, /* tp_descr_get */
863 0, /* tp_descr_set */
864 0, /* tp_dictoffset */
865 (initproc)Compressor_init, /* tp_init */
866 0, /* tp_alloc */
867 PyType_GenericNew, /* tp_new */
868};
869
870
871/* LZMADecompressor class. */
872
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100873/* Decompress data of length d->lzs.avail_in in d->lzs.next_in. The output
874 buffer is allocated dynamically and returned. At most max_length bytes are
875 returned, so some of the input may not be consumed. d->lzs.next_in and
876 d->lzs.avail_in are updated to reflect the consumed input. */
877static PyObject*
878decompress_buf(Decompressor *d, Py_ssize_t max_length)
Larry Hastingsf256c222014-01-25 21:30:37 -0800879{
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100880 Py_ssize_t data_size = 0;
Larry Hastingsf256c222014-01-25 21:30:37 -0800881 PyObject *result;
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100882 lzma_stream *lzs = &d->lzs;
Serhiy Storchaka009b8112015-03-18 21:53:15 +0200883
Serhiy Storchaka04f17f12016-10-31 08:30:09 +0200884 if (lzs->avail_in == 0)
885 return PyBytes_FromStringAndSize(NULL, 0);
886
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100887 if (max_length < 0 || max_length >= INITIAL_BUFFER_SIZE)
888 result = PyBytes_FromStringAndSize(NULL, INITIAL_BUFFER_SIZE);
889 else
890 result = PyBytes_FromStringAndSize(NULL, max_length);
Larry Hastingsf256c222014-01-25 21:30:37 -0800891 if (result == NULL)
892 return NULL;
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100893
894 lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result);
895 lzs->avail_out = PyBytes_GET_SIZE(result);
Serhiy Storchaka009b8112015-03-18 21:53:15 +0200896
Larry Hastingsf256c222014-01-25 21:30:37 -0800897 for (;;) {
898 lzma_ret lzret;
899
900 Py_BEGIN_ALLOW_THREADS
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100901 lzret = lzma_code(lzs, LZMA_RUN);
902 data_size = (char *)lzs->next_out - PyBytes_AS_STRING(result);
Larry Hastingsf256c222014-01-25 21:30:37 -0800903 Py_END_ALLOW_THREADS
904 if (catch_lzma_error(lzret))
905 goto error;
906 if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK)
907 d->check = lzma_get_check(&d->lzs);
908 if (lzret == LZMA_STREAM_END) {
909 d->eof = 1;
Larry Hastingsf256c222014-01-25 21:30:37 -0800910 break;
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100911 } else if (lzs->avail_in == 0) {
Larry Hastingsf256c222014-01-25 21:30:37 -0800912 break;
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100913 } else if (lzs->avail_out == 0) {
914 if (data_size == max_length)
915 break;
916 if (grow_buffer(&result, max_length) == -1)
Larry Hastingsf256c222014-01-25 21:30:37 -0800917 goto error;
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100918 lzs->next_out = (uint8_t *)PyBytes_AS_STRING(result) + data_size;
919 lzs->avail_out = PyBytes_GET_SIZE(result) - data_size;
Larry Hastingsf256c222014-01-25 21:30:37 -0800920 }
921 }
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100922 if (data_size != PyBytes_GET_SIZE(result))
Larry Hastingsf256c222014-01-25 21:30:37 -0800923 if (_PyBytes_Resize(&result, data_size) == -1)
924 goto error;
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100925
926 return result;
Serhiy Storchaka009b8112015-03-18 21:53:15 +0200927
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100928error:
929 Py_XDECREF(result);
930 return NULL;
931}
932
933static PyObject *
934decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length)
935{
936 char input_buffer_in_use;
937 PyObject *result;
938 lzma_stream *lzs = &d->lzs;
Serhiy Storchaka009b8112015-03-18 21:53:15 +0200939
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100940 /* Prepend unconsumed input if necessary */
941 if (lzs->next_in != NULL) {
942 size_t avail_now, avail_total;
Serhiy Storchaka009b8112015-03-18 21:53:15 +0200943
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100944 /* Number of bytes we can append to input buffer */
945 avail_now = (d->input_buffer + d->input_buffer_size)
946 - (lzs->next_in + lzs->avail_in);
947
948 /* Number of bytes we can append if we move existing
949 contents to beginning of buffer (overwriting
950 consumed input) */
951 avail_total = d->input_buffer_size - lzs->avail_in;
952
953 if (avail_total < len) {
954 size_t offset = lzs->next_in - d->input_buffer;
955 uint8_t *tmp;
956 size_t new_size = d->input_buffer_size + len - avail_now;
957
958 /* Assign to temporary variable first, so we don't
959 lose address of allocated buffer if realloc fails */
960 tmp = PyMem_Realloc(d->input_buffer, new_size);
961 if (tmp == NULL) {
962 PyErr_SetNone(PyExc_MemoryError);
963 return NULL;
964 }
965 d->input_buffer = tmp;
966 d->input_buffer_size = new_size;
Serhiy Storchaka009b8112015-03-18 21:53:15 +0200967
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100968 lzs->next_in = d->input_buffer + offset;
969 }
970 else if (avail_now < len) {
971 memmove(d->input_buffer, lzs->next_in,
972 lzs->avail_in);
973 lzs->next_in = d->input_buffer;
974 }
975 memcpy((void*)(lzs->next_in + lzs->avail_in), data, len);
976 lzs->avail_in += len;
977 input_buffer_in_use = 1;
978 }
979 else {
980 lzs->next_in = data;
981 lzs->avail_in = len;
982 input_buffer_in_use = 0;
983 }
984
985 result = decompress_buf(d, max_length);
Serhiy Storchakac0b70372016-09-27 20:14:26 +0300986 if (result == NULL) {
987 lzs->next_in = NULL;
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100988 return NULL;
Serhiy Storchakac0b70372016-09-27 20:14:26 +0300989 }
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100990
991 if (d->eof) {
992 d->needs_input = 0;
993 if (lzs->avail_in > 0) {
Serhiy Storchaka48842712016-04-06 09:45:48 +0300994 Py_XSETREF(d->unused_data,
Serhiy Storchaka4a1e70f2015-12-27 12:36:18 +0200995 PyBytes_FromStringAndSize((char *)lzs->next_in, lzs->avail_in));
Antoine Pitrou26795ba2015-01-17 16:22:18 +0100996 if (d->unused_data == NULL)
997 goto error;
998 }
999 }
1000 else if (lzs->avail_in == 0) {
1001 lzs->next_in = NULL;
1002 d->needs_input = 1;
1003 }
1004 else {
1005 d->needs_input = 0;
1006
1007 /* If we did not use the input buffer, we now have
1008 to copy the tail from the caller's buffer into the
1009 input buffer */
1010 if (!input_buffer_in_use) {
1011
1012 /* Discard buffer if it's too small
1013 (resizing it may needlessly copy the current contents) */
1014 if (d->input_buffer != NULL &&
1015 d->input_buffer_size < lzs->avail_in) {
1016 PyMem_Free(d->input_buffer);
1017 d->input_buffer = NULL;
1018 }
1019
1020 /* Allocate if necessary */
1021 if (d->input_buffer == NULL) {
1022 d->input_buffer = PyMem_Malloc(lzs->avail_in);
1023 if (d->input_buffer == NULL) {
1024 PyErr_SetNone(PyExc_MemoryError);
1025 goto error;
1026 }
1027 d->input_buffer_size = lzs->avail_in;
1028 }
1029
1030 /* Copy tail */
1031 memcpy(d->input_buffer, lzs->next_in, lzs->avail_in);
1032 lzs->next_in = d->input_buffer;
1033 }
1034 }
Serhiy Storchaka009b8112015-03-18 21:53:15 +02001035
Larry Hastingsf256c222014-01-25 21:30:37 -08001036 return result;
1037
1038error:
1039 Py_XDECREF(result);
1040 return NULL;
1041}
1042
1043/*[clinic input]
1044_lzma.LZMADecompressor.decompress
1045
Larry Hastingsf256c222014-01-25 21:30:37 -08001046 data: Py_buffer
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001047 max_length: Py_ssize_t=-1
Larry Hastingsf256c222014-01-25 21:30:37 -08001048
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001049Decompress *data*, returning uncompressed data as bytes.
Larry Hastingsf256c222014-01-25 21:30:37 -08001050
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001051If *max_length* is nonnegative, returns at most *max_length* bytes of
1052decompressed data. If this limit is reached and further output can be
1053produced, *self.needs_input* will be set to ``False``. In this case, the next
1054call to *decompress()* may provide *data* as b'' to obtain more of the output.
Larry Hastingsf256c222014-01-25 21:30:37 -08001055
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001056If all of the input data was decompressed and returned (either because this
1057was less than *max_length* bytes, or because *max_length* was negative),
1058*self.needs_input* will be set to True.
1059
1060Attempting to decompress data after the end of stream is reached raises an
1061EOFError. Any data found after the end of the stream is ignored and saved in
1062the unused_data attribute.
Larry Hastingsf256c222014-01-25 21:30:37 -08001063[clinic start generated code]*/
1064
1065static PyObject *
Larry Hastings89964c42015-04-14 18:07:59 -04001066_lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data,
1067 Py_ssize_t max_length)
Serhiy Storchaka7a9579c2016-05-02 13:45:20 +03001068/*[clinic end generated code: output=ef4e20ec7122241d input=60c1f135820e309d]*/
Larry Hastingsf256c222014-01-25 21:30:37 -08001069{
1070 PyObject *result = NULL;
1071
1072 ACQUIRE_LOCK(self);
1073 if (self->eof)
1074 PyErr_SetString(PyExc_EOFError, "Already at end of stream");
1075 else
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001076 result = decompress(self, data->buf, data->len, max_length);
Larry Hastingsf256c222014-01-25 21:30:37 -08001077 RELEASE_LOCK(self);
1078 return result;
1079}
1080
1081static PyObject *
1082Decompressor_getstate(Decompressor *self, PyObject *noargs)
1083{
1084 PyErr_Format(PyExc_TypeError, "cannot serialize '%s' object",
1085 Py_TYPE(self)->tp_name);
1086 return NULL;
1087}
1088
1089static int
1090Decompressor_init_raw(lzma_stream *lzs, PyObject *filterspecs)
1091{
1092 lzma_filter filters[LZMA_FILTERS_MAX + 1];
1093 lzma_ret lzret;
1094
1095 if (parse_filter_chain_spec(filters, filterspecs) == -1)
1096 return -1;
1097 lzret = lzma_raw_decoder(lzs, filters);
1098 free_filter_chain(filters);
1099 if (catch_lzma_error(lzret))
1100 return -1;
1101 else
1102 return 0;
1103}
1104
1105/*[clinic input]
1106_lzma.LZMADecompressor.__init__
1107
Larry Hastingsf256c222014-01-25 21:30:37 -08001108 format: int(c_default="FORMAT_AUTO") = FORMAT_AUTO
1109 Specifies the container format of the input stream. If this is
1110 FORMAT_AUTO (the default), the decompressor will automatically detect
1111 whether the input is FORMAT_XZ or FORMAT_ALONE. Streams created with
1112 FORMAT_RAW cannot be autodetected.
1113
1114 memlimit: object = None
1115 Limit the amount of memory used by the decompressor. This will cause
1116 decompression to fail if the input cannot be decompressed within the
1117 given limit.
1118
1119 filters: object = None
1120 A custom filter chain. This argument is required for FORMAT_RAW, and
1121 not accepted with any other format. When provided, this should be a
1122 sequence of dicts, each indicating the ID and options for a single
1123 filter.
1124
1125Create a decompressor object for decompressing data incrementally.
1126
1127For one-shot decompression, use the decompress() function instead.
1128[clinic start generated code]*/
1129
1130static int
Larry Hastings89964c42015-04-14 18:07:59 -04001131_lzma_LZMADecompressor___init___impl(Decompressor *self, int format,
1132 PyObject *memlimit, PyObject *filters)
Serhiy Storchaka7a9579c2016-05-02 13:45:20 +03001133/*[clinic end generated code: output=3e1821f8aa36564c input=81fe684a6c2f8a27]*/
Larry Hastingsf256c222014-01-25 21:30:37 -08001134{
1135 const uint32_t decoder_flags = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK;
1136 uint64_t memlimit_ = UINT64_MAX;
1137 lzma_ret lzret;
1138
1139 if (memlimit != Py_None) {
1140 if (format == FORMAT_RAW) {
1141 PyErr_SetString(PyExc_ValueError,
1142 "Cannot specify memory limit with FORMAT_RAW");
1143 return -1;
1144 }
1145 memlimit_ = PyLong_AsUnsignedLongLong(memlimit);
1146 if (PyErr_Occurred())
1147 return -1;
1148 }
1149
1150 if (format == FORMAT_RAW && filters == Py_None) {
1151 PyErr_SetString(PyExc_ValueError,
1152 "Must specify filters for FORMAT_RAW");
1153 return -1;
1154 } else if (format != FORMAT_RAW && filters != Py_None) {
1155 PyErr_SetString(PyExc_ValueError,
1156 "Cannot specify filters except with FORMAT_RAW");
1157 return -1;
1158 }
1159
1160 self->alloc.opaque = NULL;
1161 self->alloc.alloc = PyLzma_Malloc;
1162 self->alloc.free = PyLzma_Free;
1163 self->lzs.allocator = &self->alloc;
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001164 self->lzs.next_in = NULL;
Larry Hastingsf256c222014-01-25 21:30:37 -08001165
Victor Stinner9b7cf752018-06-23 10:35:23 +02001166 PyThread_type_lock lock = PyThread_allocate_lock();
1167 if (lock == NULL) {
Larry Hastingsf256c222014-01-25 21:30:37 -08001168 PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock");
1169 return -1;
1170 }
Victor Stinner9b7cf752018-06-23 10:35:23 +02001171 if (self->lock != NULL) {
1172 PyThread_free_lock(self->lock);
1173 }
1174 self->lock = lock;
Larry Hastingsf256c222014-01-25 21:30:37 -08001175
1176 self->check = LZMA_CHECK_UNKNOWN;
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001177 self->needs_input = 1;
1178 self->input_buffer = NULL;
1179 self->input_buffer_size = 0;
Oren Milmand019bc82018-02-13 12:28:33 +02001180 Py_XSETREF(self->unused_data, PyBytes_FromStringAndSize(NULL, 0));
Larry Hastingsf256c222014-01-25 21:30:37 -08001181 if (self->unused_data == NULL)
1182 goto error;
1183
1184 switch (format) {
1185 case FORMAT_AUTO:
1186 lzret = lzma_auto_decoder(&self->lzs, memlimit_, decoder_flags);
1187 if (catch_lzma_error(lzret))
1188 break;
1189 return 0;
1190
1191 case FORMAT_XZ:
1192 lzret = lzma_stream_decoder(&self->lzs, memlimit_, decoder_flags);
1193 if (catch_lzma_error(lzret))
1194 break;
1195 return 0;
1196
1197 case FORMAT_ALONE:
1198 self->check = LZMA_CHECK_NONE;
1199 lzret = lzma_alone_decoder(&self->lzs, memlimit_);
1200 if (catch_lzma_error(lzret))
1201 break;
1202 return 0;
1203
1204 case FORMAT_RAW:
1205 self->check = LZMA_CHECK_NONE;
1206 if (Decompressor_init_raw(&self->lzs, filters) == -1)
1207 break;
1208 return 0;
1209
1210 default:
1211 PyErr_Format(PyExc_ValueError,
1212 "Invalid container format: %d", format);
1213 break;
1214 }
1215
1216error:
1217 Py_CLEAR(self->unused_data);
Larry Hastingsf256c222014-01-25 21:30:37 -08001218 PyThread_free_lock(self->lock);
1219 self->lock = NULL;
Larry Hastingsf256c222014-01-25 21:30:37 -08001220 return -1;
1221}
1222
1223static void
1224Decompressor_dealloc(Decompressor *self)
1225{
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001226 if(self->input_buffer != NULL)
1227 PyMem_Free(self->input_buffer);
Serhiy Storchaka009b8112015-03-18 21:53:15 +02001228
Larry Hastingsf256c222014-01-25 21:30:37 -08001229 lzma_end(&self->lzs);
1230 Py_CLEAR(self->unused_data);
Larry Hastingsf256c222014-01-25 21:30:37 -08001231 if (self->lock != NULL)
1232 PyThread_free_lock(self->lock);
Larry Hastingsf256c222014-01-25 21:30:37 -08001233 Py_TYPE(self)->tp_free((PyObject *)self);
1234}
1235
1236static PyMethodDef Decompressor_methods[] = {
1237 _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF
1238 {"__getstate__", (PyCFunction)Decompressor_getstate, METH_NOARGS},
1239 {NULL}
1240};
1241
1242PyDoc_STRVAR(Decompressor_check_doc,
1243"ID of the integrity check used by the input stream.");
1244
1245PyDoc_STRVAR(Decompressor_eof_doc,
1246"True if the end-of-stream marker has been reached.");
1247
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001248PyDoc_STRVAR(Decompressor_needs_input_doc,
1249"True if more input is needed before more decompressed data can be produced.");
1250
Larry Hastingsf256c222014-01-25 21:30:37 -08001251PyDoc_STRVAR(Decompressor_unused_data_doc,
1252"Data found after the end of the compressed stream.");
1253
1254static PyMemberDef Decompressor_members[] = {
1255 {"check", T_INT, offsetof(Decompressor, check), READONLY,
1256 Decompressor_check_doc},
1257 {"eof", T_BOOL, offsetof(Decompressor, eof), READONLY,
1258 Decompressor_eof_doc},
Antoine Pitrou26795ba2015-01-17 16:22:18 +01001259 {"needs_input", T_BOOL, offsetof(Decompressor, needs_input), READONLY,
1260 Decompressor_needs_input_doc},
Larry Hastingsf256c222014-01-25 21:30:37 -08001261 {"unused_data", T_OBJECT_EX, offsetof(Decompressor, unused_data), READONLY,
1262 Decompressor_unused_data_doc},
1263 {NULL}
1264};
1265
1266static PyTypeObject Decompressor_type = {
1267 PyVarObject_HEAD_INIT(NULL, 0)
1268 "_lzma.LZMADecompressor", /* tp_name */
1269 sizeof(Decompressor), /* tp_basicsize */
1270 0, /* tp_itemsize */
1271 (destructor)Decompressor_dealloc, /* tp_dealloc */
1272 0, /* tp_print */
1273 0, /* tp_getattr */
1274 0, /* tp_setattr */
1275 0, /* tp_reserved */
1276 0, /* tp_repr */
1277 0, /* tp_as_number */
1278 0, /* tp_as_sequence */
1279 0, /* tp_as_mapping */
1280 0, /* tp_hash */
1281 0, /* tp_call */
1282 0, /* tp_str */
1283 0, /* tp_getattro */
1284 0, /* tp_setattro */
1285 0, /* tp_as_buffer */
1286 Py_TPFLAGS_DEFAULT, /* tp_flags */
1287 _lzma_LZMADecompressor___init____doc__, /* tp_doc */
1288 0, /* tp_traverse */
1289 0, /* tp_clear */
1290 0, /* tp_richcompare */
1291 0, /* tp_weaklistoffset */
1292 0, /* tp_iter */
1293 0, /* tp_iternext */
1294 Decompressor_methods, /* tp_methods */
1295 Decompressor_members, /* tp_members */
1296 0, /* tp_getset */
1297 0, /* tp_base */
1298 0, /* tp_dict */
1299 0, /* tp_descr_get */
1300 0, /* tp_descr_set */
1301 0, /* tp_dictoffset */
1302 _lzma_LZMADecompressor___init__, /* tp_init */
1303 0, /* tp_alloc */
1304 PyType_GenericNew, /* tp_new */
1305};
1306
1307
1308/* Module-level functions. */
1309
1310/*[clinic input]
1311_lzma.is_check_supported
1312 check_id: int
1313 /
1314
1315Test whether the given integrity check is supported.
1316
1317Always returns True for CHECK_NONE and CHECK_CRC32.
1318[clinic start generated code]*/
1319
1320static PyObject *
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03001321_lzma_is_check_supported_impl(PyObject *module, int check_id)
1322/*[clinic end generated code: output=e4f14ba3ce2ad0a5 input=5518297b97b2318f]*/
Larry Hastingsf256c222014-01-25 21:30:37 -08001323{
1324 return PyBool_FromLong(lzma_check_is_supported(check_id));
1325}
1326
1327
1328/*[clinic input]
1329_lzma._encode_filter_properties
1330 filter: lzma_filter(c_default="{LZMA_VLI_UNKNOWN, NULL}")
1331 /
1332
1333Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).
1334
1335The result does not include the filter ID itself, only the options.
1336[clinic start generated code]*/
1337
1338static PyObject *
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03001339_lzma__encode_filter_properties_impl(PyObject *module, lzma_filter filter)
1340/*[clinic end generated code: output=5c93c8e14e7be5a8 input=d4c64f1b557c77d4]*/
Larry Hastingsf256c222014-01-25 21:30:37 -08001341{
1342 lzma_ret lzret;
1343 uint32_t encoded_size;
1344 PyObject *result = NULL;
1345
1346 lzret = lzma_properties_size(&encoded_size, &filter);
1347 if (catch_lzma_error(lzret))
1348 goto error;
1349
1350 result = PyBytes_FromStringAndSize(NULL, encoded_size);
1351 if (result == NULL)
1352 goto error;
1353
1354 lzret = lzma_properties_encode(
1355 &filter, (uint8_t *)PyBytes_AS_STRING(result));
1356 if (catch_lzma_error(lzret))
1357 goto error;
1358
1359 return result;
1360
1361error:
1362 Py_XDECREF(result);
1363 return NULL;
1364}
1365
1366
1367/*[clinic input]
1368_lzma._decode_filter_properties
1369 filter_id: lzma_vli
1370 encoded_props: Py_buffer
1371 /
1372
1373Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).
1374
1375The result does not include the filter ID itself, only the options.
1376[clinic start generated code]*/
1377
1378static PyObject *
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03001379_lzma__decode_filter_properties_impl(PyObject *module, lzma_vli filter_id,
Larry Hastings89964c42015-04-14 18:07:59 -04001380 Py_buffer *encoded_props)
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03001381/*[clinic end generated code: output=714fd2ef565d5c60 input=246410800782160c]*/
Larry Hastingsf256c222014-01-25 21:30:37 -08001382{
1383 lzma_filter filter;
1384 lzma_ret lzret;
1385 PyObject *result = NULL;
1386 filter.id = filter_id;
1387
1388 lzret = lzma_properties_decode(
1389 &filter, NULL, encoded_props->buf, encoded_props->len);
1390 if (catch_lzma_error(lzret))
1391 return NULL;
1392
1393 result = build_filter_spec(&filter);
1394
1395 /* We use vanilla free() here instead of PyMem_Free() - filter.options was
1396 allocated by lzma_properties_decode() using the default allocator. */
1397 free(filter.options);
1398 return result;
1399}
1400
1401
1402/* Module initialization. */
1403
1404static PyMethodDef module_methods[] = {
1405 _LZMA_IS_CHECK_SUPPORTED_METHODDEF
1406 _LZMA__ENCODE_FILTER_PROPERTIES_METHODDEF
1407 _LZMA__DECODE_FILTER_PROPERTIES_METHODDEF
1408 {NULL}
1409};
1410
1411static PyModuleDef _lzmamodule = {
1412 PyModuleDef_HEAD_INIT,
1413 "_lzma",
1414 NULL,
1415 -1,
1416 module_methods,
1417 NULL,
1418 NULL,
1419 NULL,
1420 NULL,
1421};
1422
1423/* Some of our constants are more than 32 bits wide, so PyModule_AddIntConstant
1424 would not work correctly on platforms with 32-bit longs. */
1425static int
Benjamin Petersonaf580df2016-09-06 10:46:49 -07001426module_add_int_constant(PyObject *m, const char *name, long long value)
Larry Hastingsf256c222014-01-25 21:30:37 -08001427{
1428 PyObject *o = PyLong_FromLongLong(value);
1429 if (o == NULL)
1430 return -1;
1431 if (PyModule_AddObject(m, name, o) == 0)
1432 return 0;
1433 Py_DECREF(o);
1434 return -1;
1435}
1436
1437#define ADD_INT_PREFIX_MACRO(m, macro) \
1438 module_add_int_constant(m, #macro, LZMA_ ## macro)
1439
1440PyMODINIT_FUNC
1441PyInit__lzma(void)
1442{
1443 PyObject *m;
1444
1445 empty_tuple = PyTuple_New(0);
1446 if (empty_tuple == NULL)
1447 return NULL;
1448
1449 m = PyModule_Create(&_lzmamodule);
1450 if (m == NULL)
1451 return NULL;
1452
1453 if (PyModule_AddIntMacro(m, FORMAT_AUTO) == -1 ||
1454 PyModule_AddIntMacro(m, FORMAT_XZ) == -1 ||
1455 PyModule_AddIntMacro(m, FORMAT_ALONE) == -1 ||
1456 PyModule_AddIntMacro(m, FORMAT_RAW) == -1 ||
1457 ADD_INT_PREFIX_MACRO(m, CHECK_NONE) == -1 ||
1458 ADD_INT_PREFIX_MACRO(m, CHECK_CRC32) == -1 ||
1459 ADD_INT_PREFIX_MACRO(m, CHECK_CRC64) == -1 ||
1460 ADD_INT_PREFIX_MACRO(m, CHECK_SHA256) == -1 ||
1461 ADD_INT_PREFIX_MACRO(m, CHECK_ID_MAX) == -1 ||
1462 ADD_INT_PREFIX_MACRO(m, CHECK_UNKNOWN) == -1 ||
1463 ADD_INT_PREFIX_MACRO(m, FILTER_LZMA1) == -1 ||
1464 ADD_INT_PREFIX_MACRO(m, FILTER_LZMA2) == -1 ||
1465 ADD_INT_PREFIX_MACRO(m, FILTER_DELTA) == -1 ||
1466 ADD_INT_PREFIX_MACRO(m, FILTER_X86) == -1 ||
1467 ADD_INT_PREFIX_MACRO(m, FILTER_IA64) == -1 ||
1468 ADD_INT_PREFIX_MACRO(m, FILTER_ARM) == -1 ||
1469 ADD_INT_PREFIX_MACRO(m, FILTER_ARMTHUMB) == -1 ||
1470 ADD_INT_PREFIX_MACRO(m, FILTER_SPARC) == -1 ||
1471 ADD_INT_PREFIX_MACRO(m, FILTER_POWERPC) == -1 ||
1472 ADD_INT_PREFIX_MACRO(m, MF_HC3) == -1 ||
1473 ADD_INT_PREFIX_MACRO(m, MF_HC4) == -1 ||
1474 ADD_INT_PREFIX_MACRO(m, MF_BT2) == -1 ||
1475 ADD_INT_PREFIX_MACRO(m, MF_BT3) == -1 ||
1476 ADD_INT_PREFIX_MACRO(m, MF_BT4) == -1 ||
1477 ADD_INT_PREFIX_MACRO(m, MODE_FAST) == -1 ||
1478 ADD_INT_PREFIX_MACRO(m, MODE_NORMAL) == -1 ||
1479 ADD_INT_PREFIX_MACRO(m, PRESET_DEFAULT) == -1 ||
1480 ADD_INT_PREFIX_MACRO(m, PRESET_EXTREME) == -1)
1481 return NULL;
1482
1483 Error = PyErr_NewExceptionWithDoc(
1484 "_lzma.LZMAError", "Call to liblzma failed.", NULL, NULL);
1485 if (Error == NULL)
1486 return NULL;
1487 Py_INCREF(Error);
1488 if (PyModule_AddObject(m, "LZMAError", Error) == -1)
1489 return NULL;
1490
1491 if (PyType_Ready(&Compressor_type) == -1)
1492 return NULL;
1493 Py_INCREF(&Compressor_type);
1494 if (PyModule_AddObject(m, "LZMACompressor",
1495 (PyObject *)&Compressor_type) == -1)
1496 return NULL;
1497
1498 if (PyType_Ready(&Decompressor_type) == -1)
1499 return NULL;
1500 Py_INCREF(&Decompressor_type);
1501 if (PyModule_AddObject(m, "LZMADecompressor",
1502 (PyObject *)&Decompressor_type) == -1)
1503 return NULL;
1504
1505 return m;
1506}