From 30bf0f51335e87812ffeb54e9437f0b6a1514d67 Mon Sep 17 00:00:00 2001 From: "Juan J. Martinez" Date: Tue, 6 Sep 2022 07:37:20 +0100 Subject: Updated rasm to 1.7 --- tools/rasm/lzsa-master/src/dictionary.c | 101 ++ tools/rasm/lzsa-master/src/dictionary.h | 64 + tools/rasm/lzsa-master/src/expand_block_v1.c | 224 ++++ tools/rasm/lzsa-master/src/expand_block_v1.h | 49 + tools/rasm/lzsa-master/src/expand_block_v2.c | 253 ++++ tools/rasm/lzsa-master/src/expand_block_v2.h | 49 + tools/rasm/lzsa-master/src/expand_context.c | 76 ++ tools/rasm/lzsa-master/src/expand_context.h | 61 + tools/rasm/lzsa-master/src/expand_inmem.c | 163 +++ tools/rasm/lzsa-master/src/expand_inmem.h | 70 + tools/rasm/lzsa-master/src/expand_streaming.c | 236 ++++ tools/rasm/lzsa-master/src/expand_streaming.h | 86 ++ tools/rasm/lzsa-master/src/format.h | 51 + tools/rasm/lzsa-master/src/frame.c | 189 +++ tools/rasm/lzsa-master/src/frame.h | 122 ++ tools/rasm/lzsa-master/src/lib.h | 95 ++ .../rasm/lzsa-master/src/libdivsufsort/.gitignore | 32 + .../lzsa-master/src/libdivsufsort/CHANGELOG.md | 21 + .../lzsa-master/src/libdivsufsort/CMakeLists.txt | 99 ++ .../CMakeModules/AppendCompilerFlags.cmake | 38 + .../CMakeModules/CheckFunctionKeywords.cmake | 15 + .../src/libdivsufsort/CMakeModules/CheckLFS.cmake | 109 ++ .../libdivsufsort/CMakeModules/ProjectCPack.cmake | 38 + .../CMakeModules/cmake_uninstall.cmake.in | 36 + tools/rasm/lzsa-master/src/libdivsufsort/LICENSE | 21 + tools/rasm/lzsa-master/src/libdivsufsort/README.md | 140 ++ .../lzsa-master/src/libdivsufsort/VERSION.cmake | 23 + .../src/libdivsufsort/examples/CMakeLists.txt | 11 + .../lzsa-master/src/libdivsufsort/examples/bwt.c | 220 ++++ .../src/libdivsufsort/examples/mksary.c | 193 +++ .../src/libdivsufsort/examples/sasearch.c | 165 +++ .../src/libdivsufsort/examples/suftest.c | 164 +++ .../lzsa-master/src/libdivsufsort/examples/unbwt.c | 207 +++ .../src/libdivsufsort/include/CMakeLists.txt | 162 +++ .../src/libdivsufsort/include/config.h.cmake | 81 ++ .../src/libdivsufsort/include/divsufsort.h | 189 +++ .../src/libdivsufsort/include/divsufsort.h.cmake | 180 +++ .../src/libdivsufsort/include/divsufsort_config.h | 9 + .../src/libdivsufsort/include/divsufsort_private.h | 205 +++ .../src/libdivsufsort/include/lfs.h.cmake | 56 + .../src/libdivsufsort/pkgconfig/CMakeLists.txt | 9 + .../libdivsufsort/pkgconfig/libdivsufsort.pc.cmake | 11 + tools/rasm/lzsa-master/src/lzsa.c | 1109 ++++++++++++++++ tools/rasm/lzsa-master/src/matchfinder.c | 361 ++++++ tools/rasm/lzsa-master/src/matchfinder.h | 91 ++ tools/rasm/lzsa-master/src/shrink_block_v1.c | 710 ++++++++++ tools/rasm/lzsa-master/src/shrink_block_v1.h | 53 + tools/rasm/lzsa-master/src/shrink_block_v2.c | 1371 ++++++++++++++++++++ tools/rasm/lzsa-master/src/shrink_block_v2.h | 53 + tools/rasm/lzsa-master/src/shrink_context.c | 254 ++++ tools/rasm/lzsa-master/src/shrink_context.h | 182 +++ tools/rasm/lzsa-master/src/shrink_inmem.c | 185 +++ tools/rasm/lzsa-master/src/shrink_inmem.h | 71 + tools/rasm/lzsa-master/src/shrink_streaming.c | 320 +++++ tools/rasm/lzsa-master/src/shrink_streaming.h | 99 ++ tools/rasm/lzsa-master/src/stream.c | 111 ++ tools/rasm/lzsa-master/src/stream.h | 103 ++ 57 files changed, 9396 insertions(+) create mode 100644 tools/rasm/lzsa-master/src/dictionary.c create mode 100644 tools/rasm/lzsa-master/src/dictionary.h create mode 100644 tools/rasm/lzsa-master/src/expand_block_v1.c create mode 100644 tools/rasm/lzsa-master/src/expand_block_v1.h create mode 100644 tools/rasm/lzsa-master/src/expand_block_v2.c create mode 100644 tools/rasm/lzsa-master/src/expand_block_v2.h create mode 100644 tools/rasm/lzsa-master/src/expand_context.c create mode 100644 tools/rasm/lzsa-master/src/expand_context.h create mode 100644 tools/rasm/lzsa-master/src/expand_inmem.c create mode 100644 tools/rasm/lzsa-master/src/expand_inmem.h create mode 100644 tools/rasm/lzsa-master/src/expand_streaming.c create mode 100644 tools/rasm/lzsa-master/src/expand_streaming.h create mode 100755 tools/rasm/lzsa-master/src/format.h create mode 100644 tools/rasm/lzsa-master/src/frame.c create mode 100644 tools/rasm/lzsa-master/src/frame.h create mode 100755 tools/rasm/lzsa-master/src/lib.h create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/.gitignore create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/CHANGELOG.md create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/CMakeLists.txt create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/AppendCompilerFlags.cmake create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/CheckFunctionKeywords.cmake create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/CheckLFS.cmake create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/ProjectCPack.cmake create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/cmake_uninstall.cmake.in create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/LICENSE create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/README.md create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/VERSION.cmake create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/examples/CMakeLists.txt create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/examples/bwt.c create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/examples/mksary.c create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/examples/sasearch.c create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/examples/suftest.c create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/examples/unbwt.c create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/include/CMakeLists.txt create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/include/config.h.cmake create mode 100755 tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort.h create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort.h.cmake create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort_config.h create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort_private.h create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/include/lfs.h.cmake create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/pkgconfig/CMakeLists.txt create mode 100644 tools/rasm/lzsa-master/src/libdivsufsort/pkgconfig/libdivsufsort.pc.cmake create mode 100755 tools/rasm/lzsa-master/src/lzsa.c create mode 100644 tools/rasm/lzsa-master/src/matchfinder.c create mode 100644 tools/rasm/lzsa-master/src/matchfinder.h create mode 100644 tools/rasm/lzsa-master/src/shrink_block_v1.c create mode 100644 tools/rasm/lzsa-master/src/shrink_block_v1.h create mode 100644 tools/rasm/lzsa-master/src/shrink_block_v2.c create mode 100644 tools/rasm/lzsa-master/src/shrink_block_v2.h create mode 100644 tools/rasm/lzsa-master/src/shrink_context.c create mode 100644 tools/rasm/lzsa-master/src/shrink_context.h create mode 100644 tools/rasm/lzsa-master/src/shrink_inmem.c create mode 100644 tools/rasm/lzsa-master/src/shrink_inmem.h create mode 100644 tools/rasm/lzsa-master/src/shrink_streaming.c create mode 100644 tools/rasm/lzsa-master/src/shrink_streaming.h create mode 100644 tools/rasm/lzsa-master/src/stream.c create mode 100644 tools/rasm/lzsa-master/src/stream.h (limited to 'tools/rasm/lzsa-master') diff --git a/tools/rasm/lzsa-master/src/dictionary.c b/tools/rasm/lzsa-master/src/dictionary.c new file mode 100644 index 0000000..fbc4f52 --- /dev/null +++ b/tools/rasm/lzsa-master/src/dictionary.c @@ -0,0 +1,101 @@ +/* + * dictionary.c - dictionary implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "format.h" +#include "lib.h" + +/** + * Load dictionary contents + * + * @param pszDictionaryFilename name of dictionary file, or NULL for none + * @param ppDictionaryData pointer to returned dictionary contents, or NULL for none + * @param pDictionaryDataSize pointer to returned size of dictionary contents, or 0 + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +int lzsa_dictionary_load(const char *pszDictionaryFilename, void **ppDictionaryData, int *pDictionaryDataSize) { + unsigned char *pDictionaryData = NULL; + int nDictionaryDataSize = 0; + + if (pszDictionaryFilename) { + pDictionaryData = (unsigned char *)malloc(BLOCK_SIZE); + if (!pDictionaryData) { + return LZSA_ERROR_MEMORY; + } + + FILE *pDictionaryFile = fopen(pszDictionaryFilename, "rb"); + if (!pDictionaryFile) { + free(pDictionaryData); + pDictionaryData = NULL; + return LZSA_ERROR_DICTIONARY; + } + + fseek(pDictionaryFile, 0, SEEK_END); +#ifdef _WIN32 + __int64 nDictionaryFileSize = _ftelli64(pDictionaryFile); +#else + off_t nDictionaryFileSize = ftello(pDictionaryFile); +#endif + if (nDictionaryFileSize > BLOCK_SIZE) { + /* Use the last BLOCK_SIZE bytes of the dictionary */ + fseek(pDictionaryFile, -BLOCK_SIZE, SEEK_END); + } + else { + fseek(pDictionaryFile, 0, SEEK_SET); + } + + nDictionaryDataSize = (int)fread(pDictionaryData, 1, BLOCK_SIZE, pDictionaryFile); + if (nDictionaryDataSize < 0) + nDictionaryDataSize = 0; + + fclose(pDictionaryFile); + pDictionaryFile = NULL; + } + + *ppDictionaryData = pDictionaryData; + *pDictionaryDataSize = nDictionaryDataSize; + return LZSA_OK; +} + +/** + * Free dictionary contents + * + * @param ppDictionaryData pointer to pointer to dictionary contents + */ +void lzsa_dictionary_free(void **ppDictionaryData) { + if (*ppDictionaryData) { + free(*ppDictionaryData); + *ppDictionaryData = NULL; + } +} diff --git a/tools/rasm/lzsa-master/src/dictionary.h b/tools/rasm/lzsa-master/src/dictionary.h new file mode 100644 index 0000000..a363564 --- /dev/null +++ b/tools/rasm/lzsa-master/src/dictionary.h @@ -0,0 +1,64 @@ +/* + * dictionary.h - dictionary definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _DICTIONARY_H +#define _DICTIONARY_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Load dictionary contents + * + * @param pszDictionaryFilename name of dictionary file, or NULL for none + * @param ppDictionaryData pointer to returned dictionary contents, or NULL for none + * @param pDictionaryDataSize pointer to returned size of dictionary contents, or 0 + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +int lzsa_dictionary_load(const char *pszDictionaryFilename, void **ppDictionaryData, int *pDictionaryDataSize); + +/** + * Free dictionary contents + * + * @param ppDictionaryData pointer to pointer to dictionary contents + */ +void lzsa_dictionary_free(void **ppDictionaryData); + +#ifdef __cplusplus +} +#endif + +#endif /* _DICTIONARY_H */ diff --git a/tools/rasm/lzsa-master/src/expand_block_v1.c b/tools/rasm/lzsa-master/src/expand_block_v1.c new file mode 100644 index 0000000..c248fb3 --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_block_v1.c @@ -0,0 +1,224 @@ +/* + * expand_block_v1.c - LZSA1 block decompressor implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "format.h" +#include "expand_block_v1.h" + +#ifdef _MSC_VER +#define FORCE_INLINE __forceinline +#else /* _MSC_VER */ +#define FORCE_INLINE __attribute__((always_inline)) +#endif /* _MSC_VER */ + +static inline FORCE_INLINE int lzsa_build_literals_len_v1(const unsigned char **ppInBlock, const unsigned char *pInBlockEnd, unsigned int *nLiterals) { + unsigned int nByte; + const unsigned char *pInBlock = *ppInBlock; + + if (pInBlock < pInBlockEnd) { + nByte = *pInBlock++; + (*nLiterals) += nByte; + + if (nByte == 250) { + if (pInBlock < pInBlockEnd) { + (*nLiterals) = 256 + ((unsigned int)*pInBlock++); + } + else { + return -1; + } + } + else if (nByte == 249) { + if ((pInBlock + 1) < pInBlockEnd) { + (*nLiterals) = ((unsigned int)*pInBlock++); + (*nLiterals) |= (((unsigned int)*pInBlock++) << 8); + } + else { + return -1; + } + } + + *ppInBlock = pInBlock; + return 0; + } + else { + return -1; + } +} + +static inline FORCE_INLINE int lzsa_build_match_len_v1(const unsigned char **ppInBlock, const unsigned char *pInBlockEnd, unsigned int *nMatchLen) { + unsigned int nByte; + const unsigned char *pInBlock = *ppInBlock; + + if (pInBlock < pInBlockEnd) { + nByte = *pInBlock++; + (*nMatchLen) += nByte; + + if (nByte == 239) { + if (pInBlock < pInBlockEnd) { + (*nMatchLen) = 256 + ((unsigned int)*pInBlock++); + } + else { + return -1; + } + } + else if (nByte == 238) { + if ((pInBlock + 1) < pInBlockEnd) { + (*nMatchLen) = ((unsigned int)*pInBlock++); + (*nMatchLen) |= (((unsigned int)*pInBlock++) << 8); + } + else { + return -1; + } + } + + *ppInBlock = pInBlock; + return 0; + } + else { + return -1; + } +} + +/** + * Decompress one LZSA1 data block + * + * @param pInBlock pointer to compressed data + * @param nBlockSize size of compressed data, in bytes + * @param pOutData pointer to output decompression buffer (previously decompressed bytes + room for decompressing this block) + * @param nOutDataOffset starting index of where to store decompressed bytes in output buffer (and size of previously decompressed bytes) + * @param nBlockMaxSize total size of output decompression buffer, in bytes + * + * @return size of decompressed data in bytes, or -1 for error + */ +int lzsa_decompressor_expand_block_v1(const unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize) { + const unsigned char *pInBlockEnd = pInBlock + nBlockSize; + unsigned char *pCurOutData = pOutData + nOutDataOffset; + const unsigned char *pOutDataEnd = pCurOutData + nBlockMaxSize; + const unsigned char *pOutDataFastEnd = pOutDataEnd - 18; + + while (pInBlock < pInBlockEnd) { + const unsigned char token = *pInBlock++; + unsigned int nLiterals = (unsigned int)((token & 0x70) >> 4); + + if (nLiterals != LITERALS_RUN_LEN_V1 && (pInBlock + 8) <= pInBlockEnd && pCurOutData < pOutDataFastEnd) { + memcpy(pCurOutData, pInBlock, 8); + pInBlock += nLiterals; + pCurOutData += nLiterals; + } + else { + if (nLiterals == LITERALS_RUN_LEN_V1) { + if (lzsa_build_literals_len_v1(&pInBlock, pInBlockEnd, &nLiterals)) + return -1; + } + + if (nLiterals != 0) { + if ((pInBlock + nLiterals) <= pInBlockEnd && + (pCurOutData + nLiterals) <= pOutDataEnd) { + memcpy(pCurOutData, pInBlock, nLiterals); + pInBlock += nLiterals; + pCurOutData += nLiterals; + } + else { + return -1; + } + } + } + + if ((pInBlock + 1) < pInBlockEnd) { /* The last token in the block does not include match information */ + unsigned int nMatchOffset; + + nMatchOffset = ((unsigned int)(*pInBlock++)) ^ 0xff; + if (token & 0x80) { + nMatchOffset |= (((unsigned int)(*pInBlock++)) << 8) ^ 0xff00; + } + nMatchOffset++; + + const unsigned char *pSrc = pCurOutData - nMatchOffset; + if (pSrc >= pOutData) { + unsigned int nMatchLen = (unsigned int)(token & 0x0f); + if (nMatchLen != MATCH_RUN_LEN_V1 && nMatchOffset >= 8 && pCurOutData < pOutDataFastEnd && (pSrc + 18) <= pOutDataEnd) { + memcpy(pCurOutData, pSrc, 8); + memcpy(pCurOutData + 8, pSrc + 8, 8); + memcpy(pCurOutData + 16, pSrc + 16, 2); + pCurOutData += (MIN_MATCH_SIZE_V1 + nMatchLen); + } + else { + nMatchLen += MIN_MATCH_SIZE_V1; + if (nMatchLen == (MATCH_RUN_LEN_V1 + MIN_MATCH_SIZE_V1)) { + if (lzsa_build_match_len_v1(&pInBlock, pInBlockEnd, &nMatchLen)) + return -1; + if (nMatchLen == 0) + break; + } + + if ((pSrc + nMatchLen) <= pOutDataEnd) { + if ((pCurOutData + nMatchLen) <= pOutDataEnd) { + /* Do a deterministic, left to right byte copy instead of memcpy() so as to handle overlaps */ + + if (nMatchOffset >= 16 && (pCurOutData + nMatchLen) < (pOutDataFastEnd - 15)) { + const unsigned char *pCopySrc = pSrc; + unsigned char *pCopyDst = pCurOutData; + const unsigned char *pCopyEndDst = pCurOutData + nMatchLen; + + do { + memcpy(pCopyDst, pCopySrc, 16); + pCopySrc += 16; + pCopyDst += 16; + } while (pCopyDst < pCopyEndDst); + + pCurOutData += nMatchLen; + } + else { + while (nMatchLen) { + *pCurOutData++ = *pSrc++; + nMatchLen--; + } + } + } + else { + return -1; + } + } + else { + return -1; + } + } + } + else { + return -1; + } + } + } + + return (int)(pCurOutData - (pOutData + nOutDataOffset)); +} diff --git a/tools/rasm/lzsa-master/src/expand_block_v1.h b/tools/rasm/lzsa-master/src/expand_block_v1.h new file mode 100644 index 0000000..a1d5651 --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_block_v1.h @@ -0,0 +1,49 @@ +/* + * expand_block_v1.h - LZSA1 block decompressor definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _EXPAND_BLOCK_V1_H +#define _EXPAND_BLOCK_V1_H + +/** + * Decompress one LZSA1 data block + * + * @param pInBlock pointer to compressed data + * @param nBlockSize size of compressed data, in bytes + * @param pOutData pointer to output decompression buffer (previously decompressed bytes + room for decompressing this block) + * @param nOutDataOffset starting index of where to store decompressed bytes in output buffer (and size of previously decompressed bytes) + * @param nBlockMaxSize total size of output decompression buffer, in bytes + * + * @return size of decompressed data in bytes, or -1 for error + */ +int lzsa_decompressor_expand_block_v1(const unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize); + +#endif /* _EXPAND_BLOCK_V1_H */ diff --git a/tools/rasm/lzsa-master/src/expand_block_v2.c b/tools/rasm/lzsa-master/src/expand_block_v2.c new file mode 100644 index 0000000..e024b4c --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_block_v2.c @@ -0,0 +1,253 @@ +/* + * expand_block_v2.c - LZSA2 block decompressor implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "format.h" +#include "expand_block_v2.h" + +#ifdef _MSC_VER +#define FORCE_INLINE __forceinline +#else /* _MSC_VER */ +#define FORCE_INLINE __attribute__((always_inline)) +#endif /* _MSC_VER */ + +static inline FORCE_INLINE unsigned int lzsa_get_nibble_v2(const unsigned char **ppInBlock, const unsigned char *pInBlockEnd, int *nCurNibbles, unsigned char *nibbles, unsigned int *nValue) { + if ((*nCurNibbles ^= 1) != 0) { + const unsigned char *pInBlock = *ppInBlock; + if (pInBlock < pInBlockEnd) { + (*nibbles) = *pInBlock++; + *ppInBlock = pInBlock; + (*nValue) = ((unsigned int)((*nibbles) & 0xf0)) >> 4; + return 0; + } + else { + return -1; + } + } + + (*nValue) = (unsigned int)((*nibbles) & 0x0f); + return 0; +} + +static inline FORCE_INLINE int lzsa_build_len_v2(const unsigned char **ppInBlock, const unsigned char *pInBlockEnd, int *nCurNibbles, unsigned char *nibbles, unsigned int *nLength) { + unsigned int nValue; + + if (!lzsa_get_nibble_v2(ppInBlock, pInBlockEnd, nCurNibbles, nibbles, &nValue)) { + (*nLength) += nValue; + + if (nValue == 15) { + const unsigned char *pInBlock = *ppInBlock; + + if (pInBlock < pInBlockEnd) { + (*nLength) += ((unsigned int)*pInBlock++); + + if ((*nLength) == 257) { + if ((pInBlock + 1) < pInBlockEnd) { + (*nLength) = ((unsigned int)*pInBlock++); + (*nLength) |= (((unsigned int)*pInBlock++) << 8); + } + else { + return -1; + } + } + else if ((*nLength) == 256) { + (*nLength) = 0; + } + } + else { + return -1; + } + + *ppInBlock = pInBlock; + } + + return 0; + } + else { + return -1; + } +} + +/** + * Decompress one LZSA2 data block + * + * @param pInBlock pointer to compressed data + * @param nBlockSize size of compressed data, in bytes + * @param pOutData pointer to output decompression buffer (previously decompressed bytes + room for decompressing this block) + * @param nOutDataOffset starting index of where to store decompressed bytes in output buffer (and size of previously decompressed bytes) + * @param nBlockMaxSize total size of output decompression buffer, in bytes + * + * @return size of decompressed data in bytes, or -1 for error + */ +int lzsa_decompressor_expand_block_v2(const unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize) { + const unsigned char *pInBlockEnd = pInBlock + nBlockSize; + unsigned char *pCurOutData = pOutData + nOutDataOffset; + const unsigned char *pOutDataEnd = pCurOutData + nBlockMaxSize; + const unsigned char *pOutDataFastEnd = pOutDataEnd - 20; + int nCurNibbles = 0; + unsigned char nibbles; + int nMatchOffset = 0; + + while (pInBlock < pInBlockEnd) { + const unsigned char token = *pInBlock++; + unsigned int nLiterals = (unsigned int)((token & 0x18) >> 3); + + if (nLiterals != LITERALS_RUN_LEN_V2 && (pInBlock + 4) <= pInBlockEnd && pCurOutData < pOutDataFastEnd) { + memcpy(pCurOutData, pInBlock, 4); + pInBlock += nLiterals; + pCurOutData += nLiterals; + } + else { + if (nLiterals == LITERALS_RUN_LEN_V2) { + if (lzsa_build_len_v2(&pInBlock, pInBlockEnd, &nCurNibbles, &nibbles, &nLiterals)) + return -1; + } + + if (nLiterals != 0) { + if ((pInBlock + nLiterals) <= pInBlockEnd && + (pCurOutData + nLiterals) <= pOutDataEnd) { + memcpy(pCurOutData, pInBlock, nLiterals); + pInBlock += nLiterals; + pCurOutData += nLiterals; + } + else { + return -1; + } + } + } + + if (pInBlock < pInBlockEnd) { /* The last token in the block does not include match information */ + unsigned char nOffsetMode = token & 0xc0; + unsigned int nValue; + + switch (nOffsetMode) { + case 0x00: + /* 5 bit offset */ + if (lzsa_get_nibble_v2(&pInBlock, pInBlockEnd, &nCurNibbles, &nibbles, &nValue)) + return -1; + nMatchOffset = nValue << 1; + nMatchOffset |= ((token & 0x20) >> 5); + nMatchOffset ^= 0x1e; + nMatchOffset++; + break; + + case 0x40: + /* 9 bit offset */ + nMatchOffset = (unsigned int)(*pInBlock++); + nMatchOffset |= (((unsigned int)(token & 0x20)) << 3); + nMatchOffset ^= 0x0ff; + nMatchOffset++; + break; + + case 0x80: + /* 13 bit offset */ + if (lzsa_get_nibble_v2(&pInBlock, pInBlockEnd, &nCurNibbles, &nibbles, &nValue)) + return -1; + nMatchOffset = (unsigned int)(*pInBlock++); + nMatchOffset |= (nValue << 9); + nMatchOffset |= (((unsigned int)(token & 0x20)) << 3); + nMatchOffset ^= 0x1eff; + nMatchOffset += (512 + 1); + break; + + default: + /* Check if this is a 16 bit offset or a rep-match */ + if ((token & 0x20) == 0) { + /* 16 bit offset */ + nMatchOffset = (((unsigned int)(*pInBlock++)) << 8); + if (pInBlock >= pInBlockEnd) return -1; + nMatchOffset |= (unsigned int)(*pInBlock++); + nMatchOffset ^= 0xffff; + nMatchOffset++; + } + break; + } + + const unsigned char *pSrc = pCurOutData - nMatchOffset; + if (pSrc >= pOutData) { + unsigned int nMatchLen = (unsigned int)(token & 0x07); + if (nMatchLen != MATCH_RUN_LEN_V2 && nMatchOffset >= 8 && pCurOutData < pOutDataFastEnd && (pSrc + 10) <= pOutDataEnd) { + memcpy(pCurOutData, pSrc, 8); + memcpy(pCurOutData + 8, pSrc + 8, 2); + pCurOutData += (MIN_MATCH_SIZE_V2 + nMatchLen); + } + else { + nMatchLen += MIN_MATCH_SIZE_V2; + if (nMatchLen == (MATCH_RUN_LEN_V2 + MIN_MATCH_SIZE_V2)) { + if (lzsa_build_len_v2(&pInBlock, pInBlockEnd, &nCurNibbles, &nibbles, &nMatchLen)) + return -1; + if (nMatchLen == 0) + break; + } + + if ((pSrc + nMatchLen) <= pOutDataEnd) { + if ((pCurOutData + nMatchLen) <= pOutDataEnd) { + /* Do a deterministic, left to right byte copy instead of memcpy() so as to handle overlaps */ + + if (nMatchOffset >= 16 && (pCurOutData + nMatchLen) < (pOutDataFastEnd - 15)) { + const unsigned char *pCopySrc = pSrc; + unsigned char *pCopyDst = pCurOutData; + const unsigned char *pCopyEndDst = pCurOutData + nMatchLen; + + do { + memcpy(pCopyDst, pCopySrc, 16); + pCopySrc += 16; + pCopyDst += 16; + } while (pCopyDst < pCopyEndDst); + + pCurOutData += nMatchLen; + } + else { + while (nMatchLen) { + *pCurOutData++ = *pSrc++; + nMatchLen--; + } + } + } + else { + return -1; + } + } + else { + return -1; + } + } + } + else { + return -1; + } + } + } + + return (int)(pCurOutData - (pOutData + nOutDataOffset)); +} diff --git a/tools/rasm/lzsa-master/src/expand_block_v2.h b/tools/rasm/lzsa-master/src/expand_block_v2.h new file mode 100644 index 0000000..f612f7a --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_block_v2.h @@ -0,0 +1,49 @@ +/* + * expand_block_v2.h - LZSA2 block decompressor definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _EXPAND_BLOCK_V2_H +#define _EXPAND_BLOCK_V2_H + +/** + * Decompress one LZSA2 data block + * + * @param pInBlock pointer to compressed data + * @param nBlockSize size of compressed data, in bytes + * @param pOutData pointer to output decompression buffer (previously decompressed bytes + room for decompressing this block) + * @param nOutDataOffset starting index of where to store decompressed bytes in output buffer (and size of previously decompressed bytes) + * @param nBlockMaxSize total size of output decompression buffer, in bytes + * + * @return size of decompressed data in bytes, or -1 for error + */ +int lzsa_decompressor_expand_block_v2(const unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize); + +#endif /* _EXPAND_BLOCK_V2_H */ diff --git a/tools/rasm/lzsa-master/src/expand_context.c b/tools/rasm/lzsa-master/src/expand_context.c new file mode 100644 index 0000000..b02620d --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_context.c @@ -0,0 +1,76 @@ +/* + * expand_context.h - decompressor context definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "expand_context.h" +#include "expand_block_v1.h" +#include "expand_block_v2.h" +#include "lib.h" + +/** + * Decompress one data block + * + * @param pInBlock pointer to compressed data + * @param nBlockSize size of compressed data, in bytes + * @param pOutData pointer to output decompression buffer (previously decompressed bytes + room for decompressing this block) + * @param nOutDataOffset starting index of where to store decompressed bytes in output buffer (and size of previously decompressed bytes) + * @param nBlockMaxSize total size of output decompression buffer, in bytes + * @param nFormatVersion version of format to use (1-2) + * @param nFlags compression flags (LZSA_FLAG_xxx) + * + * @return size of decompressed data in bytes, or -1 for error + */ +int lzsa_decompressor_expand_block(unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize, const int nFormatVersion, const int nFlags) { + int nDecompressedSize; + + if (nFlags & LZSA_FLAG_RAW_BACKWARD) { + lzsa_reverse_buffer(pInBlock, nBlockSize); + } + + if (nFormatVersion == 1) + nDecompressedSize = lzsa_decompressor_expand_block_v1(pInBlock, nBlockSize, pOutData, nOutDataOffset, nBlockMaxSize); + else if (nFormatVersion == 2) + nDecompressedSize = lzsa_decompressor_expand_block_v2(pInBlock, nBlockSize, pOutData, nOutDataOffset, nBlockMaxSize); + else + nDecompressedSize = -1; + + if (nDecompressedSize != -1 && (nFlags & LZSA_FLAG_RAW_BACKWARD)) { + lzsa_reverse_buffer(pOutData + nOutDataOffset, nDecompressedSize); + } + + if (nFlags & LZSA_FLAG_RAW_BACKWARD) { + lzsa_reverse_buffer(pInBlock, nBlockSize); + } + + return nDecompressedSize; +} diff --git a/tools/rasm/lzsa-master/src/expand_context.h b/tools/rasm/lzsa-master/src/expand_context.h new file mode 100644 index 0000000..698e039 --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_context.h @@ -0,0 +1,61 @@ +/* + * expand_context.h - decompressor context definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _EXPAND_CONTEXT_H +#define _EXPAND_CONTEXT_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Decompress one data block + * + * @param pInBlock pointer to compressed data + * @param nBlockSize size of compressed data, in bytes + * @param pOutData pointer to output decompression buffer (previously decompressed bytes + room for decompressing this block) + * @param nOutDataOffset starting index of where to store decompressed bytes in output buffer (and size of previously decompressed bytes) + * @param nBlockMaxSize total size of output decompression buffer, in bytes + * @param nFormatVersion version of format to use (1-2) + * @param nFlags compression flags (LZSA_FLAG_xxx) + * + * @return size of decompressed data in bytes, or -1 for error + */ +int lzsa_decompressor_expand_block(unsigned char *pInBlock, int nBlockSize, unsigned char *pOutData, int nOutDataOffset, int nBlockMaxSize, const int nFormatVersion, const int nFlags); + +#ifdef __cplusplus +} +#endif + +#endif /* _EXPAND_CONTEXT_H */ diff --git a/tools/rasm/lzsa-master/src/expand_inmem.c b/tools/rasm/lzsa-master/src/expand_inmem.c new file mode 100644 index 0000000..050bdbc --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_inmem.c @@ -0,0 +1,163 @@ +/* + * expand_inmem.c - in-memory decompression implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "expand_inmem.h" +#include "lib.h" +#include "frame.h" + +#define BLOCK_SIZE 65536 + +/** + * Get maximum decompressed size of compressed data + * + * @param pFileData compressed data + * @param nFileSize compressed size in bytes + * + * @return maximum decompressed size + */ +size_t lzsa_get_max_decompressed_size_inmem(const unsigned char *pFileData, size_t nFileSize) { + const unsigned char *pCurFileData = pFileData; + const unsigned char *pEndFileData = pCurFileData + nFileSize; + int nFormatVersion = 0; + size_t nMaxDecompressedSize = 0; + const int nHeaderSize = lzsa_get_header_size(); + + /* Check header */ + if ((pCurFileData + nHeaderSize) > pEndFileData || + lzsa_decode_header(pCurFileData, nHeaderSize, &nFormatVersion) != 0) + return -1; + + pCurFileData += nHeaderSize; + + while (pCurFileData < pEndFileData) { + unsigned int nBlockDataSize = 0; + int nIsUncompressed = 0; + const int nFrameSize = lzsa_get_frame_size(); + + /* Decode frame header */ + if ((pCurFileData + nFrameSize) > pEndFileData || + lzsa_decode_frame(pCurFileData, nFrameSize, &nBlockDataSize, &nIsUncompressed) != 0) + return -1; + pCurFileData += nFrameSize; + + if (!nBlockDataSize) + break; + + /* Add one potentially full block to the decompressed size */ + nMaxDecompressedSize += BLOCK_SIZE; + + if ((pCurFileData + nBlockDataSize) > pEndFileData) + return -1; + + pCurFileData += nBlockDataSize; + } + + return nMaxDecompressedSize; +} + +/** + * Decompress data in memory + * + * @param pFileData compressed data + * @param pOutBuffer buffer for decompressed data + * @param nFileSize compressed size in bytes + * @param nMaxOutBufferSize maximum capacity of decompression buffer + * @param nFlags compression flags (LZSA_FLAG_xxx) + * @param pFormatVersion pointer to format version, updated if this function is successful + * + * @return actual decompressed size, or -1 for error + */ +size_t lzsa_decompress_inmem(unsigned char *pFileData, unsigned char *pOutBuffer, size_t nFileSize, size_t nMaxOutBufferSize, const unsigned int nFlags, int *pFormatVersion) { + unsigned char *pCurFileData = pFileData; + const unsigned char *pEndFileData = pCurFileData + nFileSize; + unsigned char *pCurOutBuffer = pOutBuffer; + const unsigned char *pEndOutBuffer = pCurOutBuffer + nMaxOutBufferSize; + int nPreviousBlockSize; + const int nHeaderSize = lzsa_get_header_size(); + + if (nFlags & LZSA_FLAG_RAW_BLOCK) { + return (size_t)lzsa_decompressor_expand_block(pFileData, (int)nFileSize, pOutBuffer, 0, (int)nMaxOutBufferSize, *pFormatVersion, nFlags); + } + + /* Check header */ + if ((pCurFileData + nHeaderSize) > pEndFileData || + lzsa_decode_header(pCurFileData, nHeaderSize, pFormatVersion) != 0) + return -1; + + pCurFileData += nHeaderSize; + nPreviousBlockSize = 0; + + while (pCurFileData < pEndFileData) { + unsigned int nBlockDataSize = 0; + int nIsUncompressed = 0; + const int nFrameSize = lzsa_get_frame_size(); + + /* Decode frame header */ + if ((pCurFileData + nFrameSize) > pEndFileData || + lzsa_decode_frame(pCurFileData, nFrameSize, &nBlockDataSize, &nIsUncompressed) != 0) + return -1; + pCurFileData += nFrameSize; + + if (!nBlockDataSize) + break; + + if (!nIsUncompressed) { + int nDecompressedSize; + + /* Decompress block */ + if ((pCurFileData + nBlockDataSize) > pEndFileData) + return -1; + + nDecompressedSize = lzsa_decompressor_expand_block(pCurFileData, nBlockDataSize, pCurOutBuffer - nPreviousBlockSize, nPreviousBlockSize, (int)(pEndOutBuffer - pCurOutBuffer + nPreviousBlockSize), *pFormatVersion, nFlags); + if (nDecompressedSize < 0) + return -1; + + pCurOutBuffer += nDecompressedSize; + nPreviousBlockSize = nDecompressedSize; + } + else { + /* Copy uncompressed block */ + if ((pCurFileData + nBlockDataSize) > pEndFileData) + return -1; + if ((pCurOutBuffer + nBlockDataSize) > pEndOutBuffer) + return -1; + memcpy(pCurOutBuffer, pCurFileData, nBlockDataSize); + pCurOutBuffer += nBlockDataSize; + } + + pCurFileData += nBlockDataSize; + } + + return (int)(pCurOutBuffer - pOutBuffer); +} diff --git a/tools/rasm/lzsa-master/src/expand_inmem.h b/tools/rasm/lzsa-master/src/expand_inmem.h new file mode 100644 index 0000000..b52ee68 --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_inmem.h @@ -0,0 +1,70 @@ +/* + * expand_inmem.h - in-memory decompression definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _EXPAND_INMEM_H +#define _EXPAND_INMEM_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Get maximum decompressed size of compressed data + * + * @param pFileData compressed data + * @param nFileSize compressed size in bytes + * + * @return maximum decompressed size + */ +size_t lzsa_get_max_decompressed_size_inmem(const unsigned char *pFileData, size_t nFileSize); + +/** + * Decompress data in memory + * + * @param pFileData compressed data + * @param pOutBuffer buffer for decompressed data + * @param nFileSize compressed size in bytes + * @param nMaxOutBufferSize maximum capacity of decompression buffer + * @param nFlags compression flags (LZSA_FLAG_xxx) + * @param pFormatVersion pointer to format version, updated if this function is successful + * + * @return actual decompressed size, or -1 for error + */ +size_t lzsa_decompress_inmem(unsigned char *pFileData, unsigned char *pOutBuffer, size_t nFileSize, size_t nMaxOutBufferSize, const unsigned int nFlags, int *pFormatVersion); + +#ifdef __cplusplus +} +#endif + +#endif /* _EXPAND_INMEM_H */ diff --git a/tools/rasm/lzsa-master/src/expand_streaming.c b/tools/rasm/lzsa-master/src/expand_streaming.c new file mode 100644 index 0000000..b241c68 --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_streaming.c @@ -0,0 +1,236 @@ +/* + * expand_streaming.c - streaming decompression definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + + +#include +#include +#include "expand_streaming.h" +#include "format.h" +#include "frame.h" +#include "lib.h" + +/*-------------- File API -------------- */ + +/** + * Decompress file + * + * @param pszInFilename name of input(compressed) file to decompress + * @param pszOutFilename name of output(decompressed) file to generate + * @param pszDictionaryFilename name of dictionary file, or NULL for none + * @param nFlags compression flags (LZSA_FLAG_RAW_BLOCK to decompress a raw block, or 0) + * @param nFormatVersion default version of format to use (1-2). This is used when decompressing a raw block, otherwise the version is extracted from the source file + * @param pOriginalSize pointer to returned output(decompressed) size, updated when this function is successful + * @param pCompressedSize pointer to returned input(compressed) size, updated when this function is successful + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +lzsa_status_t lzsa_decompress_file(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, const unsigned int nFlags, int nFormatVersion, + long long *pOriginalSize, long long *pCompressedSize) { + lzsa_stream_t inStream, outStream; + void *pDictionaryData = NULL; + int nDictionaryDataSize = 0; + lzsa_status_t nStatus; + + if (lzsa_filestream_open(&inStream, pszInFilename, "rb") < 0) { + return LZSA_ERROR_SRC; + } + + if (lzsa_filestream_open(&outStream, pszOutFilename, "wb") < 0) { + inStream.close(&inStream); + return LZSA_ERROR_DST; + } + + nStatus = lzsa_dictionary_load(pszDictionaryFilename, &pDictionaryData, &nDictionaryDataSize); + if (nStatus) { + outStream.close(&outStream); + inStream.close(&inStream); + return nStatus; + } + + nStatus = lzsa_decompress_stream(&inStream, &outStream, pDictionaryData, nDictionaryDataSize, nFlags, nFormatVersion, pOriginalSize, pCompressedSize); + + lzsa_dictionary_free(&pDictionaryData); + outStream.close(&outStream); + inStream.close(&inStream); + + return nStatus; +} + +/*-------------- Streaming API -------------- */ + +/** + * Decompress stream + * + * @param pInStream input(compressed) stream to decompress + * @param pOutStream output(decompressed) stream to write to + * @param pDictionaryData dictionary contents, or NULL for none + * @param nDictionaryDataSize size of dictionary contents, or 0 + * @param nFlags compression flags (LZSA_FLAG_RAW_BLOCK to decompress a raw block, or 0) + * @param nFormatVersion default version of format to use (1-2). This is used when decompressing a raw block, otherwise the version is extracted from the source file + * @param pOriginalSize pointer to returned output(decompressed) size, updated when this function is successful + * @param pCompressedSize pointer to returned input(compressed) size, updated when this function is successful + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +lzsa_status_t lzsa_decompress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOutStream, const void *pDictionaryData, int nDictionaryDataSize, const unsigned int nFlags, int nFormatVersion, + long long *pOriginalSize, long long *pCompressedSize) { + long long nOriginalSize = 0LL, nCompressedSize = 0LL; + unsigned char cFrameData[16]; + unsigned char *pInBlock; + unsigned char *pOutData; + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) == 0) { + const int nHeaderSize = lzsa_get_header_size(); + + memset(cFrameData, 0, 16); + if (pInStream->read(pInStream, cFrameData, nHeaderSize) != nHeaderSize) { + return LZSA_ERROR_SRC; + } + + if (lzsa_decode_header(cFrameData, nHeaderSize, &nFormatVersion) < 0) { + return LZSA_ERROR_FORMAT; + } + + nCompressedSize += (long long)nHeaderSize; + } + + pInBlock = (unsigned char*)malloc(BLOCK_SIZE); + if (!pInBlock) { + return LZSA_ERROR_MEMORY; + } + + pOutData = (unsigned char*)malloc(BLOCK_SIZE * 2); + if (!pOutData) { + free(pInBlock); + pInBlock = NULL; + + return LZSA_ERROR_MEMORY; + } + + int nDecompressionError = 0; + int nPrevDecompressedSize = 0; + int nNumBlocks = 0; + + while (!pInStream->eof(pInStream) && !nDecompressionError) { + unsigned int nBlockSize = 0; + int nIsUncompressed = 0; + + if (nPrevDecompressedSize != 0) { + memcpy(pOutData + BLOCK_SIZE - nPrevDecompressedSize, pOutData + BLOCK_SIZE, nPrevDecompressedSize); + } + else if (nDictionaryDataSize && pDictionaryData) { + nPrevDecompressedSize = nDictionaryDataSize; + memcpy(pOutData + BLOCK_SIZE - nPrevDecompressedSize, pDictionaryData, nPrevDecompressedSize); + } + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) == 0) { + const int nFrameSize = lzsa_get_frame_size(); + + memset(cFrameData, 0, 16); + if (pInStream->read(pInStream, cFrameData, nFrameSize) == nFrameSize) { + if (lzsa_decode_frame(cFrameData, nFrameSize, &nBlockSize, &nIsUncompressed) < 0) { + nDecompressionError = LZSA_ERROR_FORMAT; + nBlockSize = 0; + } + + nCompressedSize += (long long)nFrameSize; + } + else { + nDecompressionError = LZSA_ERROR_SRC; + nBlockSize = 0; + } + } + else { + if (!nNumBlocks) + nBlockSize = BLOCK_SIZE; + else + nBlockSize = 0; + } + + if (nBlockSize != 0) { + int nDecompressedSize = 0; + + if ((int)nBlockSize > BLOCK_SIZE) { + nDecompressionError = LZSA_ERROR_FORMAT; + break; + } + size_t nReadBytes = pInStream->read(pInStream, pInBlock, nBlockSize); + if (nFlags & LZSA_FLAG_RAW_BLOCK) { + nBlockSize = (unsigned int)nReadBytes; + } + + if (nReadBytes == nBlockSize) { + nCompressedSize += (long long)nReadBytes; + + if (nIsUncompressed) { + memcpy(pOutData + BLOCK_SIZE, pInBlock, nBlockSize); + nDecompressedSize = nBlockSize; + } + else { + nDecompressedSize = lzsa_decompressor_expand_block(pInBlock, nBlockSize, pOutData, BLOCK_SIZE, BLOCK_SIZE, nFormatVersion, nFlags); + if (nDecompressedSize < 0) { + nDecompressionError = LZSA_ERROR_DECOMPRESSION; + break; + } + } + + if (nDecompressedSize != 0) { + nOriginalSize += (long long)nDecompressedSize; + + if (pOutStream->write(pOutStream, pOutData + BLOCK_SIZE, nDecompressedSize) != nDecompressedSize) + nDecompressionError = LZSA_ERROR_DST; + nPrevDecompressedSize = nDecompressedSize; + nDecompressedSize = 0; + } + } + else { + break; + } + + nNumBlocks++; + } + else { + break; + } + } + + free(pOutData); + pOutData = NULL; + + free(pInBlock); + pInBlock = NULL; + + *pOriginalSize = nOriginalSize; + *pCompressedSize = nCompressedSize; + return nDecompressionError; +} + diff --git a/tools/rasm/lzsa-master/src/expand_streaming.h b/tools/rasm/lzsa-master/src/expand_streaming.h new file mode 100644 index 0000000..30dd88d --- /dev/null +++ b/tools/rasm/lzsa-master/src/expand_streaming.h @@ -0,0 +1,86 @@ +/* + * expand_streaming.h - streaming decompression definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _EXPAND_STREAMING_H +#define _EXPAND_STREAMING_H + +#include "stream.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Forward declaration */ +typedef enum _lzsa_status_t lzsa_status_t; + +/*-------------- File API -------------- */ + +/** + * Decompress file + * + * @param pszInFilename name of input(compressed) file to decompress + * @param pszOutFilename name of output(decompressed) file to generate + * @param pszDictionaryFilename name of dictionary file, or NULL for none + * @param nFlags compression flags (LZSA_FLAG_RAW_BLOCK to decompress a raw block, or 0) + * @param nFormatVersion default version of format to use (1-2). This is used when decompressing a raw block, otherwise the version is extracted from the source file + * @param pOriginalSize pointer to returned output(decompressed) size, updated when this function is successful + * @param pCompressedSize pointer to returned input(compressed) size, updated when this function is successful + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +lzsa_status_t lzsa_decompress_file(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, const unsigned int nFlags, int nFormatVersion, + long long *pOriginalSize, long long *pCompressedSize); + +/*-------------- Streaming API -------------- */ + +/** + * Decompress stream + * + * @param pInStream input(compressed) stream to decompress + * @param pOutStream output(decompressed) stream to write to + * @param pDictionaryData dictionary contents, or NULL for none + * @param nDictionaryDataSize size of dictionary contents, or 0 + * @param nFlags compression flags (LZSA_FLAG_RAW_BLOCK to decompress a raw block, or 0) + * @param nFormatVersion default version of format to use (1-2). This is used when decompressing a raw block, otherwise the version is extracted from the source file + * @param pOriginalSize pointer to returned output(decompressed) size, updated when this function is successful + * @param pCompressedSize pointer to returned input(compressed) size, updated when this function is successful + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +lzsa_status_t lzsa_decompress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOutStream, const void *pDictionaryData, int nDictionaryDataSize, const unsigned int nFlags, int nFormatVersion, + long long *pOriginalSize, long long *pCompressedSize); + +#ifdef __cplusplus +} +#endif + +#endif /* _EXPAND_STREAMING_H */ diff --git a/tools/rasm/lzsa-master/src/format.h b/tools/rasm/lzsa-master/src/format.h new file mode 100755 index 0000000..ace4f15 --- /dev/null +++ b/tools/rasm/lzsa-master/src/format.h @@ -0,0 +1,51 @@ +/* + * format.h - byte stream format definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _FORMAT_H +#define _FORMAT_H + +#define MIN_OFFSET 1 +#define MAX_OFFSET 0xffff + +#define MAX_VARLEN 0xffff + +#define BLOCK_SIZE 65536 + +#define MIN_MATCH_SIZE_V1 3 +#define LITERALS_RUN_LEN_V1 7 +#define MATCH_RUN_LEN_V1 15 + +#define MIN_MATCH_SIZE_V2 2 +#define LITERALS_RUN_LEN_V2 3 +#define MATCH_RUN_LEN_V2 7 + +#endif /* _FORMAT_H */ diff --git a/tools/rasm/lzsa-master/src/frame.c b/tools/rasm/lzsa-master/src/frame.c new file mode 100644 index 0000000..f1d6be2 --- /dev/null +++ b/tools/rasm/lzsa-master/src/frame.c @@ -0,0 +1,189 @@ +/* + * frame.c - frame implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "frame.h" + +#define LZSA_ID_0 0x7b +#define LZSA_ID_1 0x9e + +/** + * Get compressed file header size + * + * @return file header size + */ +int lzsa_get_header_size(void) { + return 3; +} + +/** + * Get compressed frame header size + * + * @return frame header size + */ +int lzsa_get_frame_size(void) { + return 3; +} + +/** + * Encode file header + * + * @param pFrameData encoding buffer + * @param nMaxFrameDataSize max encoding buffer size, in bytes + * + * @return number of encoded bytes, or -1 for failure + */ +int lzsa_encode_header(unsigned char *pFrameData, const int nMaxFrameDataSize, int nFormatVersion) { + if (nMaxFrameDataSize >= 3 && (nFormatVersion == 1 || nFormatVersion == 2)) { + pFrameData[0] = LZSA_ID_0; /* Magic number */ + pFrameData[1] = LZSA_ID_1; + pFrameData[2] = (nFormatVersion == 2) ? 0x20 : 0; /* Format version 1 */ + + return 3; + } + else { + return -1; + } +} + +/** + * Encode compressed block frame header + * + * @param pFrameData encoding buffer + * @param nMaxFrameDataSize max encoding buffer size, in bytes + * @param nBlockDataSize compressed block's data size, in bytes + * + * @return number of encoded bytes, or -1 for failure + */ +int lzsa_encode_compressed_block_frame(unsigned char *pFrameData, const int nMaxFrameDataSize, const int nBlockDataSize) { + if (nMaxFrameDataSize >= 3 && nBlockDataSize <= 0x7fffff) { + pFrameData[0] = nBlockDataSize & 0xff; + pFrameData[1] = (nBlockDataSize >> 8) & 0xff; + pFrameData[2] = (nBlockDataSize >> 16) & 0x7f; + + return 3; + } + else { + return -1; + } +} + +/** + * Encode uncompressed block frame header + * + * @param pFrameData encoding buffer + * @param nMaxFrameDataSize max encoding buffer size, in bytes + * @param nBlockDataSize uncompressed block's data size, in bytes + * + * @return number of encoded bytes, or -1 for failure + */ +int lzsa_encode_uncompressed_block_frame(unsigned char *pFrameData, const int nMaxFrameDataSize, const int nBlockDataSize) { + if (nMaxFrameDataSize >= 3 && nBlockDataSize <= 0x7fffff) { + pFrameData[0] = nBlockDataSize & 0xff; + pFrameData[1] = (nBlockDataSize >> 8) & 0xff; + pFrameData[2] = ((nBlockDataSize >> 16) & 0x7f) | 0x80; /* Uncompressed block */ + + return 3; + } + else { + return -1; + } +} + +/** + * Encode terminal frame header + * + * @param pFrameData encoding buffer + * @param nMaxFrameDataSize max encoding buffer size, in bytes + * + * @return number of encoded bytes, or -1 for failure + */ +int lzsa_encode_footer_frame(unsigned char *pFrameData, const int nMaxFrameDataSize) { + if (nMaxFrameDataSize >= 3) { + pFrameData[0] = 0x00; /* EOD frame */ + pFrameData[1] = 0x00; + pFrameData[2] = 0x00; + + return 3; + } + else { + return -1; + } +} + +/** + * Decode file header + * + * @param pFrameData data bytes + * @param nFrameDataSize number of bytes to decode + * + * @return 0 for success, or -1 for failure + */ +int lzsa_decode_header(const unsigned char *pFrameData, const int nFrameDataSize, int *nFormatVersion) { + if (nFrameDataSize != 3 || + pFrameData[0] != LZSA_ID_0 || + pFrameData[1] != LZSA_ID_1 || + (pFrameData[2] & 0x1f) != 0 || + ((pFrameData[2] & 0xe0) != 0x00 && (pFrameData[2] & 0xe0) != 0x20)) { + return -1; + } + else { + *nFormatVersion = (pFrameData[2] & 0xe0) ? 2 : 1; + return 0; + } +} + +/** + * Decode frame header + * + * @param pFrameData data bytes + * @param nFrameDataSize number of bytes to decode + * @param nBlockSize pointer to block size, updated if this function succeeds (set to 0 if this is the terminal frame) + * @param nIsUncompressed pointer to compressed block flag, updated if this function succeeds + * + * @return 0 for success, or -1 for failure + */ +int lzsa_decode_frame(const unsigned char *pFrameData, const int nFrameDataSize, unsigned int *nBlockSize, int *nIsUncompressed) { + if (nFrameDataSize == 3) { + *nBlockSize = ((unsigned int)pFrameData[0]) | + (((unsigned int)pFrameData[1]) << 8) | + (((unsigned int)pFrameData[2]) << 16); + + *nIsUncompressed = ((*nBlockSize & 0x800000) != 0) ? 1 : 0; + *nBlockSize &= 0x7fffff; + return 0; + } + else { + return -1; + } +} diff --git a/tools/rasm/lzsa-master/src/frame.h b/tools/rasm/lzsa-master/src/frame.h new file mode 100644 index 0000000..7b5db99 --- /dev/null +++ b/tools/rasm/lzsa-master/src/frame.h @@ -0,0 +1,122 @@ +/* + * frame.h - frame definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _FRAME_H +#define _FRAME_H + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Get compressed file header size + * + * @return file header size + */ +int lzsa_get_header_size(void); + +/** + * Get compressed frame header size + * + * @return frame header size + */ +int lzsa_get_frame_size(void); + +/** + * Encode file header + * + * @param pFrameData encoding buffer + * @param nMaxFrameDataSize max encoding buffer size, in bytes + * + * @return number of encoded bytes, or -1 for failure + */ +int lzsa_encode_header(unsigned char *pFrameData, const int nMaxFrameDataSize, int nFormatVersion); + +/** + * Encode compressed block frame header + * + * @param pFrameData encoding buffer + * @param nMaxFrameDataSize max encoding buffer size, in bytes + * @param nBlockDataSize compressed block's data size, in bytes + * + * @return number of encoded bytes, or -1 for failure + */ +int lzsa_encode_compressed_block_frame(unsigned char *pFrameData, const int nMaxFrameDataSize, const int nBlockDataSize); + +/** + * Encode uncompressed block frame header + * + * @param pFrameData encoding buffer + * @param nMaxFrameDataSize max encoding buffer size, in bytes + * @param nBlockDataSize uncompressed block's data size, in bytes + * + * @return number of encoded bytes, or -1 for failure + */ +int lzsa_encode_uncompressed_block_frame(unsigned char *pFrameData, const int nMaxFrameDataSize, const int nBlockDataSize); + +/** + * Encode terminal frame header + * + * @param pFrameData encoding buffer + * @param nMaxFrameDataSize max encoding buffer size, in bytes + * + * @return number of encoded bytes, or -1 for failure + */ +int lzsa_encode_footer_frame(unsigned char *pFrameData, const int nMaxFrameDataSize); + +/** + * Decode file header + * + * @param pFrameData data bytes + * @param nFrameDataSize number of bytes to decode + * + * @return 0 for success, or -1 for failure + */ +int lzsa_decode_header(const unsigned char *pFrameData, const int nFrameDataSize, int *nFormatVersion); + +/** + * Decode frame header + * + * @param pFrameData data bytes + * @param nFrameDataSize number of bytes to decode + * @param nBlockSize pointer to block size, updated if this function succeeds (set to 0 if this is the terminal frame) + * @param nIsUncompressed pointer to compressed block flag, updated if this function succeeds + * + * @return 0 for success, or -1 for failure + */ +int lzsa_decode_frame(const unsigned char *pFrameData, const int nFrameDataSize, unsigned int *nBlockSize, int *nIsUncompressed); + +#ifdef __cplusplus +} +#endif + +#endif /* _FRAME_H */ diff --git a/tools/rasm/lzsa-master/src/lib.h b/tools/rasm/lzsa-master/src/lib.h new file mode 100755 index 0000000..2520b13 --- /dev/null +++ b/tools/rasm/lzsa-master/src/lib.h @@ -0,0 +1,95 @@ +/* + * lib.h - LZSA library definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _LIB_H +#define _LIB_H + +#include "stream.h" +#include "dictionary.h" +#include "frame.h" +#include "format.h" +#include "shrink_context.h" +#include "shrink_streaming.h" +#include "shrink_inmem.h" +#include "expand_context.h" +#include "expand_streaming.h" +#include "expand_inmem.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** High level status for compression and decompression */ +typedef enum _lzsa_status_t { + LZSA_OK = 0, /**< Success */ + LZSA_ERROR_SRC, /**< Error reading input */ + LZSA_ERROR_DST, /**< Error reading output */ + LZSA_ERROR_DICTIONARY, /**< Error reading dictionary */ + LZSA_ERROR_MEMORY, /**< Out of memory */ + + /* Compression-specific status codes */ + LZSA_ERROR_COMPRESSION, /**< Internal compression error */ + LZSA_ERROR_RAW_TOOLARGE, /**< Input is too large to be compressed to a raw block */ + LZSA_ERROR_RAW_UNCOMPRESSED, /**< Input is incompressible and raw blocks don't support uncompressed data */ + + /* Decompression-specific status codes */ + LZSA_ERROR_FORMAT, /**< Invalid input format or magic number when decompressing */ + LZSA_ERROR_DECOMPRESSION /**< Internal decompression error */ +} lzsa_status_t; + +/* Compression flags */ +#define LZSA_FLAG_FAVOR_RATIO (1<<0) /**< 1 to compress with the best ratio, 0 to trade some compression ratio for extra decompression speed */ +#define LZSA_FLAG_RAW_BLOCK (1<<1) /**< 1 to emit raw block */ +#define LZSA_FLAG_RAW_BACKWARD (1<<2) /**< 1 to compress or decompress raw block backward */ + +/** + * Reverse bytes in the specified buffer + * + * @param pBuffer pointer to buffer whose contents are to be reversed + * @param nBufferSize size of buffer in bytes + */ +static inline void lzsa_reverse_buffer(unsigned char *pBuffer, const int nBufferSize) { + int nMidPoint = nBufferSize / 2; + int i, j; + + for (i = 0, j = nBufferSize - 1; i < nMidPoint; i++, j--) { + unsigned char c = pBuffer[i]; + pBuffer[i] = pBuffer[j]; + pBuffer[j] = c; + } +} + +#ifdef __cplusplus +} +#endif + +#endif /* _LIB_H */ diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/.gitignore b/tools/rasm/lzsa-master/src/libdivsufsort/.gitignore new file mode 100644 index 0000000..a0a0666 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/.gitignore @@ -0,0 +1,32 @@ +# Object files +*.o +*.ko +*.obj +*.elf + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# CMake files/directories +build/ diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/CHANGELOG.md b/tools/rasm/lzsa-master/src/libdivsufsort/CHANGELOG.md new file mode 100644 index 0000000..fe9d004 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/CHANGELOG.md @@ -0,0 +1,21 @@ +# libdivsufsort Change Log + +See full changelog at: https://github.com/y-256/libdivsufsort/commits + +## [2.0.1] - 2010-11-11 +### Fixed +* Wrong variable used in `divbwt` function +* Enclose some string variables with double quotation marks in include/CMakeLists.txt +* Fix typo in include/CMakeLists.txt + +## 2.0.0 - 2008-08-23 +### Changed +* Switch the build system to [CMake](http://www.cmake.org/) +* Improve the performance of the suffix-sorting algorithm + +### Added +* OpenMP support +* 64-bit version of divsufsort + +[Unreleased]: https://github.com/y-256/libdivsufsort/compare/2.0.1...HEAD +[2.0.1]: https://github.com/y-256/libdivsufsort/compare/2.0.0...2.0.1 diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/CMakeLists.txt b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeLists.txt new file mode 100644 index 0000000..7859943 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeLists.txt @@ -0,0 +1,99 @@ +### cmake file for building libdivsufsort Package ### +cmake_minimum_required(VERSION 2.4.4) +set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") +include(AppendCompilerFlags) + +## Project information ## +project(libdivsufsort C) +set(PROJECT_VENDOR "Yuta Mori") +set(PROJECT_CONTACT "yuta.256@gmail.com") +set(PROJECT_URL "https://github.com/y-256/libdivsufsort") +set(PROJECT_DESCRIPTION "A lightweight suffix sorting library") +include(VERSION.cmake) + +## CPack configuration ## +set(CPACK_GENERATOR "TGZ;TBZ2;ZIP") +set(CPACK_SOURCE_GENERATOR "TGZ;TBZ2;ZIP") +include(ProjectCPack) + +## Project options ## +option(BUILD_SHARED_LIBS "Set to OFF to build static libraries" ON) +option(BUILD_EXAMPLES "Build examples" ON) +option(BUILD_DIVSUFSORT64 "Build libdivsufsort64" OFF) +option(USE_OPENMP "Use OpenMP for parallelization" OFF) +option(WITH_LFS "Enable Large File Support" ON) + +## Installation directories ## +set(LIB_SUFFIX "" CACHE STRING "Define suffix of directory name (32 or 64)") + +set(CMAKE_INSTALL_RUNTIMEDIR "" CACHE PATH "Specify the output directory for dll runtimes (default is bin)") +if(NOT CMAKE_INSTALL_RUNTIMEDIR) + set(CMAKE_INSTALL_RUNTIMEDIR "${CMAKE_INSTALL_PREFIX}/bin") +endif(NOT CMAKE_INSTALL_RUNTIMEDIR) + +set(CMAKE_INSTALL_LIBDIR "" CACHE PATH "Specify the output directory for libraries (default is lib)") +if(NOT CMAKE_INSTALL_LIBDIR) + set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}") +endif(NOT CMAKE_INSTALL_LIBDIR) + +set(CMAKE_INSTALL_INCLUDEDIR "" CACHE PATH "Specify the output directory for header files (default is include)") +if(NOT CMAKE_INSTALL_INCLUDEDIR) + set(CMAKE_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include") +endif(NOT CMAKE_INSTALL_INCLUDEDIR) + +set(CMAKE_INSTALL_PKGCONFIGDIR "" CACHE PATH "Specify the output directory for pkgconfig files (default is lib/pkgconfig)") +if(NOT CMAKE_INSTALL_PKGCONFIGDIR) + set(CMAKE_INSTALL_PKGCONFIGDIR "${CMAKE_INSTALL_LIBDIR}/pkgconfig") +endif(NOT CMAKE_INSTALL_PKGCONFIGDIR) + +## Build type ## +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release") +elseif(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_VERBOSE_MAKEFILE ON) +endif(NOT CMAKE_BUILD_TYPE) + +## Compiler options ## +if(MSVC) + append_c_compiler_flags("/W4" "VC" CMAKE_C_FLAGS) + append_c_compiler_flags("/Oi;/Ot;/Ox;/Oy" "VC" CMAKE_C_FLAGS_RELEASE) + if(USE_OPENMP) + append_c_compiler_flags("/openmp" "VC" CMAKE_C_FLAGS) + endif(USE_OPENMP) +elseif(BORLAND) + append_c_compiler_flags("-w" "BCC" CMAKE_C_FLAGS) + append_c_compiler_flags("-Oi;-Og;-Os;-Ov;-Ox" "BCC" CMAKE_C_FLAGS_RELEASE) +else(MSVC) + if(CMAKE_COMPILER_IS_GNUCC) + append_c_compiler_flags("-Wall" "GCC" CMAKE_C_FLAGS) + append_c_compiler_flags("-fomit-frame-pointer" "GCC" CMAKE_C_FLAGS_RELEASE) + if(USE_OPENMP) + append_c_compiler_flags("-fopenmp" "GCC" CMAKE_C_FLAGS) + endif(USE_OPENMP) + else(CMAKE_COMPILER_IS_GNUCC) + append_c_compiler_flags("-Wall" "UNKNOWN" CMAKE_C_FLAGS) + append_c_compiler_flags("-fomit-frame-pointer" "UNKNOWN" CMAKE_C_FLAGS_RELEASE) + if(USE_OPENMP) + append_c_compiler_flags("-fopenmp;-openmp;-omp" "UNKNOWN" CMAKE_C_FLAGS) + endif(USE_OPENMP) + endif(CMAKE_COMPILER_IS_GNUCC) +endif(MSVC) + +## Add definitions ## +add_definitions(-DHAVE_CONFIG_H=1 -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS) + +## Add subdirectories ## +add_subdirectory(pkgconfig) +add_subdirectory(include) +add_subdirectory(lib) +if(BUILD_EXAMPLES) + add_subdirectory(examples) +endif(BUILD_EXAMPLES) + +## Add 'uninstall' target ## +CONFIGURE_FILE( + "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/cmake_uninstall.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/CMakeModules/cmake_uninstall.cmake" + IMMEDIATE @ONLY) +ADD_CUSTOM_TARGET(uninstall + "${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_BINARY_DIR}/CMakeModules/cmake_uninstall.cmake") diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/AppendCompilerFlags.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/AppendCompilerFlags.cmake new file mode 100644 index 0000000..58d3f99 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/AppendCompilerFlags.cmake @@ -0,0 +1,38 @@ +include(CheckCSourceCompiles) +include(CheckCXXSourceCompiles) + +macro(append_c_compiler_flags _flags _name _result) + set(SAFE_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) + string(REGEX REPLACE "[-+/ ]" "_" cname "${_name}") + string(TOUPPER "${cname}" cname) + foreach(flag ${_flags}) + string(REGEX REPLACE "^[-+/ ]+(.*)[-+/ ]*$" "\\1" flagname "${flag}") + string(REGEX REPLACE "[-+/ ]" "_" flagname "${flagname}") + string(TOUPPER "${flagname}" flagname) + set(have_flag "HAVE_${cname}_${flagname}") + set(CMAKE_REQUIRED_FLAGS "${flag}") + check_c_source_compiles("int main() { return 0; }" ${have_flag}) + if(${have_flag}) + set(${_result} "${${_result}} ${flag}") + endif(${have_flag}) + endforeach(flag) + set(CMAKE_REQUIRED_FLAGS ${SAFE_CMAKE_REQUIRED_FLAGS}) +endmacro(append_c_compiler_flags) + +macro(append_cxx_compiler_flags _flags _name _result) + set(SAFE_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) + string(REGEX REPLACE "[-+/ ]" "_" cname "${_name}") + string(TOUPPER "${cname}" cname) + foreach(flag ${_flags}) + string(REGEX REPLACE "^[-+/ ]+(.*)[-+/ ]*$" "\\1" flagname "${flag}") + string(REGEX REPLACE "[-+/ ]" "_" flagname "${flagname}") + string(TOUPPER "${flagname}" flagname) + set(have_flag "HAVE_${cname}_${flagname}") + set(CMAKE_REQUIRED_FLAGS "${flag}") + check_cxx_source_compiles("int main() { return 0; }" ${have_flag}) + if(${have_flag}) + set(${_result} "${${_result}} ${flag}") + endif(${have_flag}) + endforeach(flag) + set(CMAKE_REQUIRED_FLAGS ${SAFE_CMAKE_REQUIRED_FLAGS}) +endmacro(append_cxx_compiler_flags) diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/CheckFunctionKeywords.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/CheckFunctionKeywords.cmake new file mode 100644 index 0000000..44601fd --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/CheckFunctionKeywords.cmake @@ -0,0 +1,15 @@ +include(CheckCSourceCompiles) + +macro(check_function_keywords _wordlist) + set(${_result} "") + foreach(flag ${_wordlist}) + string(REGEX REPLACE "[-+/ ()]" "_" flagname "${flag}") + string(TOUPPER "${flagname}" flagname) + set(have_flag "HAVE_${flagname}") + check_c_source_compiles("${flag} void func(); void func() { } int main() { func(); return 0; }" ${have_flag}) + if(${have_flag} AND NOT ${_result}) + set(${_result} "${flag}") +# break() + endif(${have_flag} AND NOT ${_result}) + endforeach(flag) +endmacro(check_function_keywords) diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/CheckLFS.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/CheckLFS.cmake new file mode 100644 index 0000000..e2b0099 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/CheckLFS.cmake @@ -0,0 +1,109 @@ +## Checks for large file support ## +include(CheckIncludeFile) +include(CheckSymbolExists) +include(CheckTypeSize) + +macro(check_lfs _isenable) + set(LFS_OFF_T "") + set(LFS_FOPEN "") + set(LFS_FSEEK "") + set(LFS_FTELL "") + set(LFS_PRID "") + + if(${_isenable}) + set(SAFE_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}") + set(CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} + -D_LARGEFILE_SOURCE -D_LARGE_FILES -D_FILE_OFFSET_BITS=64 + -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS) + + check_include_file("sys/types.h" HAVE_SYS_TYPES_H) + check_include_file("inttypes.h" HAVE_INTTYPES_H) + check_include_file("stddef.h" HAVE_STDDEF_H) + check_include_file("stdint.h" HAVE_STDINT_H) + + # LFS type1: 8 <= sizeof(off_t), fseeko, ftello + check_type_size("off_t" SIZEOF_OFF_T) + if(SIZEOF_OFF_T GREATER 7) + check_symbol_exists("fseeko" "stdio.h" HAVE_FSEEKO) + check_symbol_exists("ftello" "stdio.h" HAVE_FTELLO) + if(HAVE_FSEEKO AND HAVE_FTELLO) + set(LFS_OFF_T "off_t") + set(LFS_FOPEN "fopen") + set(LFS_FSEEK "fseeko") + set(LFS_FTELL "ftello") + check_symbol_exists("PRIdMAX" "inttypes.h" HAVE_PRIDMAX) + if(HAVE_PRIDMAX) + set(LFS_PRID "PRIdMAX") + else(HAVE_PRIDMAX) + check_type_size("long" SIZEOF_LONG) + check_type_size("int" SIZEOF_INT) + if(SIZEOF_OFF_T GREATER SIZEOF_LONG) + set(LFS_PRID "\"lld\"") + elseif(SIZEOF_LONG GREATER SIZEOF_INT) + set(LFS_PRID "\"ld\"") + else(SIZEOF_OFF_T GREATER SIZEOF_LONG) + set(LFS_PRID "\"d\"") + endif(SIZEOF_OFF_T GREATER SIZEOF_LONG) + endif(HAVE_PRIDMAX) + endif(HAVE_FSEEKO AND HAVE_FTELLO) + endif(SIZEOF_OFF_T GREATER 7) + + # LFS type2: 8 <= sizeof(off64_t), fopen64, fseeko64, ftello64 + if(NOT LFS_OFF_T) + check_type_size("off64_t" SIZEOF_OFF64_T) + if(SIZEOF_OFF64_T GREATER 7) + check_symbol_exists("fopen64" "stdio.h" HAVE_FOPEN64) + check_symbol_exists("fseeko64" "stdio.h" HAVE_FSEEKO64) + check_symbol_exists("ftello64" "stdio.h" HAVE_FTELLO64) + if(HAVE_FOPEN64 AND HAVE_FSEEKO64 AND HAVE_FTELLO64) + set(LFS_OFF_T "off64_t") + set(LFS_FOPEN "fopen64") + set(LFS_FSEEK "fseeko64") + set(LFS_FTELL "ftello64") + check_symbol_exists("PRIdMAX" "inttypes.h" HAVE_PRIDMAX) + if(HAVE_PRIDMAX) + set(LFS_PRID "PRIdMAX") + else(HAVE_PRIDMAX) + check_type_size("long" SIZEOF_LONG) + check_type_size("int" SIZEOF_INT) + if(SIZEOF_OFF64_T GREATER SIZEOF_LONG) + set(LFS_PRID "\"lld\"") + elseif(SIZEOF_LONG GREATER SIZEOF_INT) + set(LFS_PRID "\"ld\"") + else(SIZEOF_OFF64_T GREATER SIZEOF_LONG) + set(LFS_PRID "\"d\"") + endif(SIZEOF_OFF64_T GREATER SIZEOF_LONG) + endif(HAVE_PRIDMAX) + endif(HAVE_FOPEN64 AND HAVE_FSEEKO64 AND HAVE_FTELLO64) + endif(SIZEOF_OFF64_T GREATER 7) + endif(NOT LFS_OFF_T) + + # LFS type3: 8 <= sizeof(__int64), _fseeki64, _ftelli64 + if(NOT LFS_OFF_T) + check_type_size("__int64" SIZEOF___INT64) + if(SIZEOF___INT64 GREATER 7) + check_symbol_exists("_fseeki64" "stdio.h" HAVE__FSEEKI64) + check_symbol_exists("_ftelli64" "stdio.h" HAVE__FTELLI64) + if(HAVE__FSEEKI64 AND HAVE__FTELLI64) + set(LFS_OFF_T "__int64") + set(LFS_FOPEN "fopen") + set(LFS_FSEEK "_fseeki64") + set(LFS_FTELL "_ftelli64") + set(LFS_PRID "\"I64d\"") + endif(HAVE__FSEEKI64 AND HAVE__FTELLI64) + endif(SIZEOF___INT64 GREATER 7) + endif(NOT LFS_OFF_T) + + set(CMAKE_REQUIRED_DEFINITIONS "${SAFE_CMAKE_REQUIRED_DEFINITIONS}") + endif(${_isenable}) + + if(NOT LFS_OFF_T) + ## not found + set(LFS_OFF_T "long") + set(LFS_FOPEN "fopen") + set(LFS_FSEEK "fseek") + set(LFS_FTELL "ftell") + set(LFS_PRID "\"ld\"") + endif(NOT LFS_OFF_T) + +endmacro(check_lfs) diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/ProjectCPack.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/ProjectCPack.cmake new file mode 100644 index 0000000..7c105f9 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/ProjectCPack.cmake @@ -0,0 +1,38 @@ +# If the cmake version includes cpack, use it +IF(EXISTS "${CMAKE_ROOT}/Modules/CPack.cmake") + SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "${PROJECT_DESCRIPTION}") + SET(CPACK_PACKAGE_VENDOR "${PROJECT_VENDOR}") + SET(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.md") + SET(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") + SET(CPACK_PACKAGE_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}") + SET(CPACK_PACKAGE_VERSION_MINOR "${PROJECT_VERSION_MINOR}") + SET(CPACK_PACKAGE_VERSION_PATCH "${PROJECT_VERSION_PATCH}") +# SET(CPACK_PACKAGE_INSTALL_DIRECTORY "${PROJECT_NAME} ${PROJECT_VERSION}") + SET(CPACK_SOURCE_PACKAGE_FILE_NAME "${PROJECT_NAME}-${PROJECT_VERSION_FULL}") + + IF(NOT DEFINED CPACK_SYSTEM_NAME) + SET(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}") + ENDIF(NOT DEFINED CPACK_SYSTEM_NAME) + + IF(${CPACK_SYSTEM_NAME} MATCHES Windows) + IF(CMAKE_CL_64) + SET(CPACK_SYSTEM_NAME win64-${CMAKE_SYSTEM_PROCESSOR}) + ELSE(CMAKE_CL_64) + SET(CPACK_SYSTEM_NAME win32-${CMAKE_SYSTEM_PROCESSOR}) + ENDIF(CMAKE_CL_64) + ENDIF(${CPACK_SYSTEM_NAME} MATCHES Windows) + + IF(NOT DEFINED CPACK_PACKAGE_FILE_NAME) + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_SOURCE_PACKAGE_FILE_NAME}-${CPACK_SYSTEM_NAME}") + ENDIF(NOT DEFINED CPACK_PACKAGE_FILE_NAME) + + SET(CPACK_PACKAGE_CONTACT "${PROJECT_CONTACT}") + IF(UNIX) + SET(CPACK_STRIP_FILES "") + SET(CPACK_SOURCE_STRIP_FILES "") +# SET(CPACK_PACKAGE_EXECUTABLES "ccmake" "CMake") + ENDIF(UNIX) + SET(CPACK_SOURCE_IGNORE_FILES "/CVS/" "/build/" "/\\\\.build/" "/\\\\.svn/" "~$") + # include CPack model once all variables are set + INCLUDE(CPack) +ENDIF(EXISTS "${CMAKE_ROOT}/Modules/CPack.cmake") diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/cmake_uninstall.cmake.in b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/cmake_uninstall.cmake.in new file mode 100644 index 0000000..8366a83 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/CMakeModules/cmake_uninstall.cmake.in @@ -0,0 +1,36 @@ +IF(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") + MESSAGE(FATAL_ERROR "Cannot find install manifest: \"@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt\"") +ENDIF(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") + +FILE(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) +STRING(REGEX REPLACE "\n" ";" files "${files}") + +SET(NUM 0) +FOREACH(file ${files}) + IF(EXISTS "$ENV{DESTDIR}${file}") + MESSAGE(STATUS "Looking for \"$ENV{DESTDIR}${file}\" - found") + SET(UNINSTALL_CHECK_${NUM} 1) + ELSE(EXISTS "$ENV{DESTDIR}${file}") + MESSAGE(STATUS "Looking for \"$ENV{DESTDIR}${file}\" - not found") + SET(UNINSTALL_CHECK_${NUM} 0) + ENDIF(EXISTS "$ENV{DESTDIR}${file}") + MATH(EXPR NUM "1 + ${NUM}") +ENDFOREACH(file) + +SET(NUM 0) +FOREACH(file ${files}) + IF(${UNINSTALL_CHECK_${NUM}}) + MESSAGE(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"") + EXEC_PROGRAM( + "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" + OUTPUT_VARIABLE rm_out + RETURN_VALUE rm_retval + ) + IF(NOT "${rm_retval}" STREQUAL 0) + MESSAGE(FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"") + ENDIF(NOT "${rm_retval}" STREQUAL 0) + ENDIF(${UNINSTALL_CHECK_${NUM}}) + MATH(EXPR NUM "1 + ${NUM}") +ENDFOREACH(file) + +FILE(REMOVE "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/LICENSE b/tools/rasm/lzsa-master/src/libdivsufsort/LICENSE new file mode 100644 index 0000000..249efa4 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2003 Yuta Mori All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/README.md b/tools/rasm/lzsa-master/src/libdivsufsort/README.md new file mode 100644 index 0000000..381a188 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/README.md @@ -0,0 +1,140 @@ +# libdivsufsort + +libdivsufsort is a software library that implements a lightweight suffix array construction algorithm. + +## News +* 2015-03-21: The project has moved from [Google Code](http://code.google.com/p/libdivsufsort/) to [GitHub](https://github.com/y-256/libdivsufsort) + +## Introduction +This library provides a simple and an efficient C API to construct a suffix array and a Burrows-Wheeler transformed string from a given string over a constant-size alphabet. +The algorithm runs in O(n log n) worst-case time using only 5n+O(1) bytes of memory space, where n is the length of +the string. + +## Build requirements +* An ANSI C Compiler (e.g. GNU GCC) +* [CMake](http://www.cmake.org/ "CMake") version 2.4.2 or newer +* CMake-supported build tool + +## Building on GNU/Linux +1. Get the source code from GitHub. You can either + * use git to clone the repository + ``` + git clone https://github.com/y-256/libdivsufsort.git + ``` + * or download a [zip file](../../archive/master.zip) directly +2. Create a `build` directory in the package source directory. +```shell +$ cd libdivsufsort +$ mkdir build +$ cd build +``` +3. Configure the package for your system. +If you want to install to a different location, change the -DCMAKE_INSTALL_PREFIX option. +```shell +$ cmake -DCMAKE_BUILD_TYPE="Release" \ +-DCMAKE_INSTALL_PREFIX="/usr/local" .. +``` +4. Compile the package. +```shell +$ make +``` +5. (Optional) Install the library and header files. +```shell +$ sudo make install +``` + +## API +```c +/* Data types */ +typedef int32_t saint_t; +typedef int32_t saidx_t; +typedef uint8_t sauchar_t; + +/* + * Constructs the suffix array of a given string. + * @param T[0..n-1] The input string. + * @param SA[0..n-1] The output array or suffixes. + * @param n The length of the given string. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +saint_t +divsufsort(const sauchar_t *T, saidx_t *SA, saidx_t n); + +/* + * Constructs the burrows-wheeler transformed string of a given string. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param A[0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +saidx_t +divbwt(const sauchar_t *T, sauchar_t *U, saidx_t *A, saidx_t n); +``` + +## Example Usage +```c +#include +#include +#include + +#include + +int main() { + // intput data + char *Text = "abracadabra"; + int n = strlen(Text); + int i, j; + + // allocate + int *SA = (int *)malloc(n * sizeof(int)); + + // sort + divsufsort((unsigned char *)Text, SA, n); + + // output + for(i = 0; i < n; ++i) { + printf("SA[%2d] = %2d: ", i, SA[i]); + for(j = SA[i]; j < n; ++j) { + printf("%c", Text[j]); + } + printf("$\n"); + } + + // deallocate + free(SA); + + return 0; +} +``` +See the [examples](examples) directory for a few other examples. + +## Benchmarks +See [Benchmarks](https://github.com/y-256/libdivsufsort/blob/wiki/SACA_Benchmarks.md) page for details. + +## License +libdivsufsort is released under the [MIT license](LICENSE "MIT license"). +> The MIT License (MIT) +> +> Copyright (c) 2003 Yuta Mori All rights reserved. +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. + +## Author +* Yuta Mori diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/VERSION.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/VERSION.cmake new file mode 100644 index 0000000..3f11ac1 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/VERSION.cmake @@ -0,0 +1,23 @@ +set(PROJECT_VERSION_MAJOR "2") +set(PROJECT_VERSION_MINOR "0") +set(PROJECT_VERSION_PATCH "2") +set(PROJECT_VERSION_EXTRA "-1") +set(PROJECT_VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}") +set(PROJECT_VERSION_FULL "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}${PROJECT_VERSION_EXTRA}") + +set(LIBRARY_VERSION "3.0.1") +set(LIBRARY_SOVERSION "3") + +## Git revision number ## +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git") + execute_process(COMMAND git describe --tags HEAD + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE GIT_DESCRIBE_TAGS ERROR_QUIET) + if(GIT_DESCRIBE_TAGS) + string(REGEX REPLACE "^v(.*)" "\\1" GIT_REVISION "${GIT_DESCRIBE_TAGS}") + string(STRIP "${GIT_REVISION}" GIT_REVISION) + if(GIT_REVISION) + set(PROJECT_VERSION_FULL "${GIT_REVISION}") + endif(GIT_REVISION) + endif(GIT_DESCRIBE_TAGS) +endif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git") diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/examples/CMakeLists.txt b/tools/rasm/lzsa-master/src/libdivsufsort/examples/CMakeLists.txt new file mode 100644 index 0000000..e801c81 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/examples/CMakeLists.txt @@ -0,0 +1,11 @@ +## Add definitions ## +add_definitions(-D_LARGEFILE_SOURCE -D_LARGE_FILES -D_FILE_OFFSET_BITS=64) + +## Targets ## +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../include" + "${CMAKE_CURRENT_BINARY_DIR}/../include") +link_directories("${CMAKE_CURRENT_BINARY_DIR}/../lib") +foreach(src suftest mksary sasearch bwt unbwt) + add_executable(${src} ${src}.c) + target_link_libraries(${src} divsufsort) +endforeach(src) diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/examples/bwt.c b/tools/rasm/lzsa-master/src/libdivsufsort/examples/bwt.c new file mode 100644 index 0000000..5a362d0 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/examples/bwt.c @@ -0,0 +1,220 @@ +/* + * bwt.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include +#if HAVE_STRING_H +# include +#endif +#if HAVE_STDLIB_H +# include +#endif +#if HAVE_MEMORY_H +# include +#endif +#if HAVE_STDDEF_H +# include +#endif +#if HAVE_STRINGS_H +# include +#endif +#if HAVE_SYS_TYPES_H +# include +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include +# include +#endif +#include +#include +#include "lfs.h" + + +static +size_t +write_int(FILE *fp, saidx_t n) { + unsigned char c[4]; + c[0] = (unsigned char)((n >> 0) & 0xff), c[1] = (unsigned char)((n >> 8) & 0xff), + c[2] = (unsigned char)((n >> 16) & 0xff), c[3] = (unsigned char)((n >> 24) & 0xff); + return fwrite(c, sizeof(unsigned char), 4, fp); +} + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "bwt, a burrows-wheeler transform program, version %s.\n", + divsufsort_version()); + fprintf(stderr, "usage: %s [-b num] INFILE OUTFILE\n", progname); + fprintf(stderr, " -b num set block size to num MiB [1..512] (default: 32)\n\n"); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp, *ofp; + const char *fname, *ofname; + sauchar_t *T; + saidx_t *SA; + LFS_OFF_T n; + size_t m; + saidx_t pidx; + clock_t start,finish; + saint_t i, blocksize = 32, needclose = 3; + + /* Check arguments. */ + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if((argc != 3) && (argc != 5)) { print_help(argv[0], EXIT_FAILURE); } + i = 1; + if(argc == 5) { + if(strcmp(argv[i], "-b") != 0) { print_help(argv[0], EXIT_FAILURE); } + blocksize = atoi(argv[i + 1]); + if(blocksize < 0) { blocksize = 1; } + else if(512 < blocksize) { blocksize = 512; } + i += 2; + } + blocksize <<= 20; + + /* Open a file for reading. */ + if(strcmp(argv[i], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&fp, fname = argv[i], "rb") != 0) { +#else + if((fp = LFS_FOPEN(fname = argv[i], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdin), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + fp = stdin; + fname = "stdin"; + needclose ^= 1; + } + i += 1; + + /* Open a file for writing. */ + if(strcmp(argv[i], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&ofp, ofname = argv[i], "wb") != 0) { +#else + if((ofp = LFS_FOPEN(ofname = argv[i], "wb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdout), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + ofp = stdout; + ofname = "stdout"; + needclose ^= 2; + } + + /* Get the file size. */ + if(LFS_FSEEK(fp, 0, SEEK_END) == 0) { + n = LFS_FTELL(fp); + rewind(fp); + if(n < 0) { + fprintf(stderr, "%s: Cannot ftell `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(0x20000000L < n) { n = 0x20000000L; } + if((blocksize == 0) || (n < blocksize)) { blocksize = (saidx_t)n; } + } else if(blocksize == 0) { blocksize = 32 << 20; } + + /* Allocate 5blocksize bytes of memory. */ + T = (sauchar_t *)malloc(blocksize * sizeof(sauchar_t)); + SA = (saidx_t *)malloc(blocksize * sizeof(saidx_t)); + if((T == NULL) || (SA == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + /* Write the blocksize. */ + if(write_int(ofp, blocksize) != 4) { + fprintf(stderr, "%s: Cannot write to `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + + fprintf(stderr, " BWT (blocksize %" PRIdSAINT_T ") ... ", blocksize); + start = clock(); + for(n = 0; 0 < (m = fread(T, sizeof(sauchar_t), blocksize, fp)); n += m) { + /* Burrows-Wheeler Transform. */ + pidx = divbwt(T, T, SA, m); + if(pidx < 0) { + fprintf(stderr, "%s (bw_transform): %s.\n", + argv[0], + (pidx == -1) ? "Invalid arguments" : "Cannot allocate memory"); + exit(EXIT_FAILURE); + } + + /* Write the bwted data. */ + if((write_int(ofp, pidx) != 4) || + (fwrite(T, sizeof(sauchar_t), m, ofp) != m)) { + fprintf(stderr, "%s: Cannot write to `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } + if(ferror(fp)) { + fprintf(stderr, "%s: Cannot read from `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + finish = clock(); + fprintf(stderr, "%" PRIdOFF_T " bytes: %.4f sec\n", + n, (double)(finish - start) / (double)CLOCKS_PER_SEC); + + /* Close files */ + if(needclose & 1) { fclose(fp); } + if(needclose & 2) { fclose(ofp); } + + /* Deallocate memory. */ + free(SA); + free(T); + + return 0; +} diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/examples/mksary.c b/tools/rasm/lzsa-master/src/libdivsufsort/examples/mksary.c new file mode 100644 index 0000000..b48177c --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/examples/mksary.c @@ -0,0 +1,193 @@ +/* + * mksary.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include +#if HAVE_STRING_H +# include +#endif +#if HAVE_STDLIB_H +# include +#endif +#if HAVE_MEMORY_H +# include +#endif +#if HAVE_STDDEF_H +# include +#endif +#if HAVE_STRINGS_H +# include +#endif +#if HAVE_SYS_TYPES_H +# include +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include +# include +#endif +#include +#include +#include "lfs.h" + + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "mksary, a simple suffix array builder, version %s.\n", + divsufsort_version()); + fprintf(stderr, "usage: %s INFILE OUTFILE\n\n", progname); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp, *ofp; + const char *fname, *ofname; + sauchar_t *T; + saidx_t *SA; + LFS_OFF_T n; + clock_t start, finish; + saint_t needclose = 3; + + /* Check arguments. */ + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if(argc != 3) { print_help(argv[0], EXIT_FAILURE); } + + /* Open a file for reading. */ + if(strcmp(argv[1], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&fp, fname = argv[1], "rb") != 0) { +#else + if((fp = LFS_FOPEN(fname = argv[1], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdin), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + fp = stdin; + fname = "stdin"; + needclose ^= 1; + } + + /* Open a file for writing. */ + if(strcmp(argv[2], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&ofp, ofname = argv[2], "wb") != 0) { +#else + if((ofp = LFS_FOPEN(ofname = argv[2], "wb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdout), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + ofp = stdout; + ofname = "stdout"; + needclose ^= 2; + } + + /* Get the file size. */ + if(LFS_FSEEK(fp, 0, SEEK_END) == 0) { + n = LFS_FTELL(fp); + rewind(fp); + if(n < 0) { + fprintf(stderr, "%s: Cannot ftell `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(0x7fffffff <= n) { + fprintf(stderr, "%s: Input file `%s' is too big.\n", argv[0], fname); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: Cannot fseek `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Allocate 5blocksize bytes of memory. */ + T = (sauchar_t *)malloc((size_t)n * sizeof(sauchar_t)); + SA = (saidx_t *)malloc((size_t)n * sizeof(saidx_t)); + if((T == NULL) || (SA == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + /* Read n bytes of data. */ + if(fread(T, sizeof(sauchar_t), (size_t)n, fp) != (size_t)n) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + fname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(needclose & 1) { fclose(fp); } + + /* Construct the suffix array. */ + fprintf(stderr, "%s: %" PRIdOFF_T " bytes ... ", fname, n); + start = clock(); + if(divsufsort(T, SA, (saidx_t)n) != 0) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + finish = clock(); + fprintf(stderr, "%.4f sec\n", (double)(finish - start) / (double)CLOCKS_PER_SEC); + + /* Write the suffix array. */ + if(fwrite(SA, sizeof(saidx_t), (size_t)n, ofp) != (size_t)n) { + fprintf(stderr, "%s: Cannot write to `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(needclose & 2) { fclose(ofp); } + + /* Deallocate memory. */ + free(SA); + free(T); + + return 0; +} diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/examples/sasearch.c b/tools/rasm/lzsa-master/src/libdivsufsort/examples/sasearch.c new file mode 100644 index 0000000..7e5ca4f --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/examples/sasearch.c @@ -0,0 +1,165 @@ +/* + * sasearch.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include +#if HAVE_STRING_H +# include +#endif +#if HAVE_STDLIB_H +# include +#endif +#if HAVE_MEMORY_H +# include +#endif +#if HAVE_STDDEF_H +# include +#endif +#if HAVE_STRINGS_H +# include +#endif +#if HAVE_SYS_TYPES_H +# include +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include +# include +#endif +#include +#include "lfs.h" + + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "sasearch, a simple SA-based full-text search tool, version %s\n", + divsufsort_version()); + fprintf(stderr, "usage: %s PATTERN FILE SAFILE\n\n", progname); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp; + const char *P; + sauchar_t *T; + saidx_t *SA; + LFS_OFF_T n; + size_t Psize; + saidx_t i, size, left; + + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if(argc != 4) { print_help(argv[0], EXIT_FAILURE); } + + P = argv[1]; + Psize = strlen(P); + + /* Open a file for reading. */ +#if HAVE_FOPEN_S + if(fopen_s(&fp, argv[2], "rb") != 0) { +#else + if((fp = LFS_FOPEN(argv[2], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], argv[2]); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Get the file size. */ + if(LFS_FSEEK(fp, 0, SEEK_END) == 0) { + n = LFS_FTELL(fp); + rewind(fp); + if(n < 0) { + fprintf(stderr, "%s: Cannot ftell `%s': ", argv[0], argv[2]); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: Cannot fseek `%s': ", argv[0], argv[2]); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Allocate 5n bytes of memory. */ + T = (sauchar_t *)malloc((size_t)n * sizeof(sauchar_t)); + SA = (saidx_t *)malloc((size_t)n * sizeof(saidx_t)); + if((T == NULL) || (SA == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + /* Read n bytes of data. */ + if(fread(T, sizeof(sauchar_t), (size_t)n, fp) != (size_t)n) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + argv[2]); + perror(NULL); + exit(EXIT_FAILURE); + } + fclose(fp); + + /* Open the SA file for reading. */ +#if HAVE_FOPEN_S + if(fopen_s(&fp, argv[3], "rb") != 0) { +#else + if((fp = LFS_FOPEN(argv[3], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], argv[3]); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Read n * sizeof(saidx_t) bytes of data. */ + if(fread(SA, sizeof(saidx_t), (size_t)n, fp) != (size_t)n) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + argv[3]); + perror(NULL); + exit(EXIT_FAILURE); + } + fclose(fp); + + /* Search and print */ + size = sa_search(T, (saidx_t)n, + (const sauchar_t *)P, (saidx_t)Psize, + SA, (saidx_t)n, &left); + for(i = 0; i < size; ++i) { + fprintf(stdout, "%" PRIdSAIDX_T "\n", SA[left + i]); + } + + /* Deallocate memory. */ + free(SA); + free(T); + + return 0; +} diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/examples/suftest.c b/tools/rasm/lzsa-master/src/libdivsufsort/examples/suftest.c new file mode 100644 index 0000000..71892ac --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/examples/suftest.c @@ -0,0 +1,164 @@ +/* + * suftest.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include +#if HAVE_STRING_H +# include +#endif +#if HAVE_STDLIB_H +# include +#endif +#if HAVE_MEMORY_H +# include +#endif +#if HAVE_STDDEF_H +# include +#endif +#if HAVE_STRINGS_H +# include +#endif +#if HAVE_SYS_TYPES_H +# include +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include +# include +#endif +#include +#include +#include "lfs.h" + + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "suftest, a suffixsort tester, version %s.\n", + divsufsort_version()); + fprintf(stderr, "usage: %s FILE\n\n", progname); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp; + const char *fname; + sauchar_t *T; + saidx_t *SA; + LFS_OFF_T n; + clock_t start, finish; + saint_t needclose = 1; + + /* Check arguments. */ + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if(argc != 2) { print_help(argv[0], EXIT_FAILURE); } + + /* Open a file for reading. */ + if(strcmp(argv[1], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&fp, fname = argv[1], "rb") != 0) { +#else + if((fp = LFS_FOPEN(fname = argv[1], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdin), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + fp = stdin; + fname = "stdin"; + needclose = 0; + } + + /* Get the file size. */ + if(LFS_FSEEK(fp, 0, SEEK_END) == 0) { + n = LFS_FTELL(fp); + rewind(fp); + if(n < 0) { + fprintf(stderr, "%s: Cannot ftell `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(0x7fffffff <= n) { + fprintf(stderr, "%s: Input file `%s' is too big.\n", argv[0], fname); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: Cannot fseek `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Allocate 5n bytes of memory. */ + T = (sauchar_t *)malloc((size_t)n * sizeof(sauchar_t)); + SA = (saidx_t *)malloc((size_t)n * sizeof(saidx_t)); + if((T == NULL) || (SA == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + /* Read n bytes of data. */ + if(fread(T, sizeof(sauchar_t), (size_t)n, fp) != (size_t)n) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + argv[1]); + perror(NULL); + exit(EXIT_FAILURE); + } + if(needclose & 1) { fclose(fp); } + + /* Construct the suffix array. */ + fprintf(stderr, "%s: %" PRIdOFF_T " bytes ... ", fname, n); + start = clock(); + if(divsufsort(T, SA, (saidx_t)n) != 0) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + finish = clock(); + fprintf(stderr, "%.4f sec\n", (double)(finish - start) / (double)CLOCKS_PER_SEC); + + /* Check the suffix array. */ + if(sufcheck(T, SA, (saidx_t)n, 1) != 0) { exit(EXIT_FAILURE); } + + /* Deallocate memory. */ + free(SA); + free(T); + + return 0; +} diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/examples/unbwt.c b/tools/rasm/lzsa-master/src/libdivsufsort/examples/unbwt.c new file mode 100644 index 0000000..c0f19e9 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/examples/unbwt.c @@ -0,0 +1,207 @@ +/* + * unbwt.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include +#if HAVE_STRING_H +# include +#endif +#if HAVE_STDLIB_H +# include +#endif +#if HAVE_MEMORY_H +# include +#endif +#if HAVE_STDDEF_H +# include +#endif +#if HAVE_STRINGS_H +# include +#endif +#if HAVE_SYS_TYPES_H +# include +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include +# include +#endif +#include +#include +#include "lfs.h" + + +static +size_t +read_int(FILE *fp, saidx_t *n) { + unsigned char c[4]; + size_t m = fread(c, sizeof(unsigned char), 4, fp); + if(m == 4) { + *n = (c[0] << 0) | (c[1] << 8) | + (c[2] << 16) | (c[3] << 24); + } + return m; +} + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "unbwt, an inverse burrows-wheeler transform program, version %s.\n", + divsufsort_version()); + fprintf(stderr, "usage: %s INFILE OUTFILE\n\n", progname); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp, *ofp; + const char *fname, *ofname; + sauchar_t *T; + saidx_t *A; + LFS_OFF_T n; + size_t m; + saidx_t pidx; + clock_t start, finish; + saint_t err, blocksize, needclose = 3; + + /* Check arguments. */ + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if(argc != 3) { print_help(argv[0], EXIT_FAILURE); } + + /* Open a file for reading. */ + if(strcmp(argv[1], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&fp, fname = argv[1], "rb") != 0) { +#else + if((fp = LFS_FOPEN(fname = argv[1], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdin), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + fp = stdin; + fname = "stdin"; + needclose ^= 1; + } + + /* Open a file for writing. */ + if(strcmp(argv[2], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&ofp, ofname = argv[2], "wb") != 0) { +#else + if((ofp = LFS_FOPEN(ofname = argv[2], "wb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdout), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + ofp = stdout; + ofname = "stdout"; + needclose ^= 2; + } + + /* Read the blocksize. */ + if(read_int(fp, &blocksize) != 4) { + fprintf(stderr, "%s: Cannot read from `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Allocate 5blocksize bytes of memory. */ + T = (sauchar_t *)malloc(blocksize * sizeof(sauchar_t)); + A = (saidx_t *)malloc(blocksize * sizeof(saidx_t)); + if((T == NULL) || (A == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "UnBWT (blocksize %" PRIdSAINT_T ") ... ", blocksize); + start = clock(); + for(n = 0; (m = read_int(fp, &pidx)) != 0; n += m) { + /* Read blocksize bytes of data. */ + if((m != 4) || ((m = fread(T, sizeof(sauchar_t), blocksize, fp)) == 0)) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + fname); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Inverse Burrows-Wheeler Transform. */ + if((err = inverse_bw_transform(T, T, A, m, pidx)) != 0) { + fprintf(stderr, "%s (reverseBWT): %s.\n", + argv[0], + (err == -1) ? "Invalid data" : "Cannot allocate memory"); + exit(EXIT_FAILURE); + } + + /* Write m bytes of data. */ + if(fwrite(T, sizeof(sauchar_t), m, ofp) != m) { + fprintf(stderr, "%s: Cannot write to `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } + if(ferror(fp)) { + fprintf(stderr, "%s: Cannot read from `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + finish = clock(); + fprintf(stderr, "%" PRIdOFF_T " bytes: %.4f sec\n", + n, (double)(finish - start) / (double)CLOCKS_PER_SEC); + + /* Close files */ + if(needclose & 1) { fclose(fp); } + if(needclose & 2) { fclose(ofp); } + + /* Deallocate memory. */ + free(A); + free(T); + + return 0; +} diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/include/CMakeLists.txt b/tools/rasm/lzsa-master/src/libdivsufsort/include/CMakeLists.txt new file mode 100644 index 0000000..37781cc --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/include/CMakeLists.txt @@ -0,0 +1,162 @@ +include(CheckIncludeFiles) +include(CheckIncludeFile) +include(CheckSymbolExists) +include(CheckTypeSize) +include(CheckFunctionKeywords) +include(CheckLFS) + +## Checks for header files ## +check_include_file("inttypes.h" HAVE_INTTYPES_H) +check_include_file("memory.h" HAVE_MEMORY_H) +check_include_file("stddef.h" HAVE_STDDEF_H) +check_include_file("stdint.h" HAVE_STDINT_H) +check_include_file("stdlib.h" HAVE_STDLIB_H) +check_include_file("string.h" HAVE_STRING_H) +check_include_file("strings.h" HAVE_STRINGS_H) +check_include_file("sys/types.h" HAVE_SYS_TYPES_H) +if(HAVE_INTTYPES_H) + set(INCFILE "#include ") +elseif(HAVE_STDINT_H) + set(INCFILE "#include ") +else(HAVE_INTTYPES_H) + set(INCFILE "") +endif(HAVE_INTTYPES_H) + +## create configuration files from .cmake file ## +if(BUILD_EXAMPLES) + ## Checks for WinIO ## + if(WIN32) + check_include_file("io.h" HAVE_IO_H) + check_include_file("fcntl.h" HAVE_FCNTL_H) + check_symbol_exists("_setmode" "io.h;fcntl.h" HAVE__SETMODE) + if(NOT HAVE__SETMODE) + check_symbol_exists("setmode" "io.h;fcntl.h" HAVE_SETMODE) + endif(NOT HAVE__SETMODE) + check_symbol_exists("_fileno" "stdio.h" HAVE__FILENO) + check_symbol_exists("fopen_s" "stdio.h" HAVE_FOPEN_S) + check_symbol_exists("_O_BINARY" "fcntl.h" HAVE__O_BINARY) + endif(WIN32) + + ## Checks for large file support ## + check_lfs(WITH_LFS) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/lfs.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/lfs.h" @ONLY) +endif(BUILD_EXAMPLES) + +## generate config.h ## +check_function_keywords("inline;__inline;__inline__;__declspec(dllexport);__declspec(dllimport)") +if(HAVE_INLINE) + set(INLINE "inline") +elseif(HAVE___INLINE) + set(INLINE "__inline") +elseif(HAVE___INLINE__) + set(INLINE "__inline__") +else(HAVE_INLINE) + set(INLINE "") +endif(HAVE_INLINE) +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/config.h") + +## Checks for types ## +# sauchar_t (8bit) +check_type_size("uint8_t" UINT8_T) +if(HAVE_UINT8_T) + set(SAUCHAR_TYPE "uint8_t") +else(HAVE_UINT8_T) + check_type_size("unsigned char" SIZEOF_UNSIGNED_CHAR) + if("${SIZEOF_UNSIGNED_CHAR}" STREQUAL "1") + set(SAUCHAR_TYPE "unsigned char") + else("${SIZEOF_UNSIGNED_CHAR}" STREQUAL "1") + message(FATAL_ERROR "Cannot find unsigned 8-bit integer type") + endif("${SIZEOF_UNSIGNED_CHAR}" STREQUAL "1") +endif(HAVE_UINT8_T) +# saint_t (32bit) +check_type_size("int32_t" INT32_T) +if(HAVE_INT32_T) + set(SAINT32_TYPE "int32_t") + check_symbol_exists("PRId32" "inttypes.h" HAVE_PRID32) + if(HAVE_PRID32) + set(SAINT32_PRId "PRId32") + else(HAVE_PRID32) + set(SAINT32_PRId "\"d\"") + endif(HAVE_PRID32) +else(HAVE_INT32_T) + check_type_size("int" SIZEOF_INT) + check_type_size("long" SIZEOF_LONG) + check_type_size("short" SIZEOF_SHORT) + check_type_size("__int32" SIZEOF___INT32) + if("${SIZEOF_INT}" STREQUAL "4") + set(SAINT32_TYPE "int") + set(SAINT32_PRId "\"d\"") + elseif("${SIZEOF_LONG}" STREQUAL "4") + set(SAINT32_TYPE "long") + set(SAINT32_PRId "\"ld\"") + elseif("${SIZEOF_SHORT}" STREQUAL "4") + set(SAINT32_TYPE "short") + set(SAINT32_PRId "\"d\"") + elseif("${SIZEOF___INT32}" STREQUAL "4") + set(SAINT32_TYPE "__int32") + set(SAINT32_PRId "\"d\"") + else("${SIZEOF_INT}" STREQUAL "4") + message(FATAL_ERROR "Cannot find 32-bit integer type") + endif("${SIZEOF_INT}" STREQUAL "4") +endif(HAVE_INT32_T) +# saint64_t (64bit) +if(BUILD_DIVSUFSORT64) + check_type_size("int64_t" INT64_T) + if(HAVE_INT64_T) + set(SAINT64_TYPE "int64_t") + check_symbol_exists("PRId64" "inttypes.h" HAVE_PRID64) + if(HAVE_PRID64) + set(SAINT64_PRId "PRId64") + else(HAVE_PRID64) + set(SAINT64_PRId "\"lld\"") + endif(HAVE_PRID64) + else(HAVE_INT64_T) + check_type_size("int" SIZEOF_INT) + check_type_size("long" SIZEOF_LONG) + check_type_size("long long" SIZEOF_LONG_LONG) + check_type_size("__int64" SIZEOF___INT64) + if("${SIZEOF_INT}" STREQUAL "8") + set(SAINT64_TYPE "int") + set(SAINT64_PRId "\"d\"") + elseif("${SIZEOF_LONG}" STREQUAL "8") + set(SAINT64_TYPE "long") + set(SAINT64_PRId "\"ld\"") + elseif("${SIZEOF_LONG_LONG}" STREQUAL "8") + set(SAINT64_TYPE "long long") + set(SAINT64_PRId "\"lld\"") + elseif("${SIZEOF___INT64}" STREQUAL "8") + set(SAINT64_TYPE "__int64") + set(SAINT64_PRId "\"I64d\"") + else("${SIZEOF_INT}" STREQUAL "8") + message(SEND_ERROR "Cannot find 64-bit integer type") + set(BUILD_DIVSUFSORT64 OFF) + endif("${SIZEOF_INT}" STREQUAL "8") + endif(HAVE_INT64_T) +endif(BUILD_DIVSUFSORT64) + +## generate divsufsort.h ## +set(DIVSUFSORT_IMPORT "") +set(DIVSUFSORT_EXPORT "") +if(BUILD_SHARED_LIBS) + if(HAVE___DECLSPEC_DLLIMPORT_) + set(DIVSUFSORT_IMPORT "__declspec(dllimport)") + endif(HAVE___DECLSPEC_DLLIMPORT_) + if(HAVE___DECLSPEC_DLLEXPORT_) + set(DIVSUFSORT_EXPORT "__declspec(dllexport)") + endif(HAVE___DECLSPEC_DLLEXPORT_) +endif(BUILD_SHARED_LIBS) +set(W64BIT "") +set(SAINDEX_TYPE "${SAINT32_TYPE}") +set(SAINDEX_PRId "${SAINT32_PRId}") +set(SAINT_PRId "${SAINT32_PRId}") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/divsufsort.h.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/divsufsort.h" @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/divsufsort.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +if(BUILD_DIVSUFSORT64) + set(W64BIT "64") + set(SAINDEX_TYPE "${SAINT64_TYPE}") + set(SAINDEX_PRId "${SAINT64_PRId}") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/divsufsort.h.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/divsufsort64.h" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/divsufsort64.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +endif(BUILD_DIVSUFSORT64) diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/include/config.h.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/include/config.h.cmake new file mode 100644 index 0000000..6a1cf47 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/include/config.h.cmake @@ -0,0 +1,81 @@ +/* + * config.h for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _CONFIG_H +#define _CONFIG_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** Define to the version of this package. **/ +#cmakedefine PROJECT_VERSION_FULL "${PROJECT_VERSION_FULL}" + +/** Define to 1 if you have the header files. **/ +#cmakedefine HAVE_INTTYPES_H 1 +#cmakedefine HAVE_STDDEF_H 1 +#cmakedefine HAVE_STDINT_H 1 +#cmakedefine HAVE_STDLIB_H 1 +#cmakedefine HAVE_STRING_H 1 +#cmakedefine HAVE_STRINGS_H 1 +#cmakedefine HAVE_MEMORY_H 1 +#cmakedefine HAVE_SYS_TYPES_H 1 + +/** for WinIO **/ +#cmakedefine HAVE_IO_H 1 +#cmakedefine HAVE_FCNTL_H 1 +#cmakedefine HAVE__SETMODE 1 +#cmakedefine HAVE_SETMODE 1 +#cmakedefine HAVE__FILENO 1 +#cmakedefine HAVE_FOPEN_S 1 +#cmakedefine HAVE__O_BINARY 1 +#ifndef HAVE__SETMODE +# if HAVE_SETMODE +# define _setmode setmode +# define HAVE__SETMODE 1 +# endif +# if HAVE__SETMODE && !HAVE__O_BINARY +# define _O_BINARY 0 +# define HAVE__O_BINARY 1 +# endif +#endif + +/** for inline **/ +#ifndef INLINE +# define INLINE @INLINE@ +#endif + +/** for VC++ warning **/ +#ifdef _MSC_VER +#pragma warning(disable: 4127) +#endif + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _CONFIG_H */ diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort.h b/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort.h new file mode 100755 index 0000000..7ebb412 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort.h @@ -0,0 +1,189 @@ +/* + * divsufsort.h for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT_H +#define _DIVSUFSORT_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#define DIVSUFSORT_API + +/*- Datatypes -*/ +#ifndef SAUCHAR_T +#define SAUCHAR_T +typedef unsigned char sauchar_t; +#endif /* SAUCHAR_T */ +#ifndef SAINT_T +#define SAINT_T +typedef int saint_t; +#endif /* SAINT_T */ +#ifndef SAIDX_T +#define SAIDX_T +typedef int saidx_t; +#endif /* SAIDX_T */ +#ifndef PRIdSAIDX_T +#define PRIdSAIDX_T "d" +#endif + +/*- divsufsort context */ +typedef struct _divsufsort_ctx_t { + saidx_t *bucket_A; + saidx_t *bucket_B; +} divsufsort_ctx_t; + +/*- Prototypes -*/ + +/** + * Initialize suffix array context + * + * @return 0 for success, or non-zero in case of an error + */ +int divsufsort_init(divsufsort_ctx_t *ctx); + +/** + * Destroy suffix array context + * + * @param ctx suffix array context to destroy + */ +void divsufsort_destroy(divsufsort_ctx_t *ctx); + +/** + * Constructs the suffix array of a given string. + * @param ctx suffix array context + * @param T[0..n-1] The input string. + * @param SA[0..n-1] The output array of suffixes. + * @param n The length of the given string. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t divsufsort_build_array(divsufsort_ctx_t *ctx, const sauchar_t *T, saidx_t *SA, saidx_t n); + +#if 0 +/** + * Constructs the burrows-wheeler transformed string of a given string. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param A[0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saidx_t +divbwt(const sauchar_t *T, sauchar_t *U, saidx_t *A, saidx_t n); + +/** + * Returns the version of the divsufsort library. + * @return The version number string. + */ +DIVSUFSORT_API +const char * +divsufsort_version(void); + + +/** + * Constructs the burrows-wheeler transformed string of a given string and suffix array. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param SA[0..n-1] The suffix array. (can be NULL) + * @param n The length of the given string. + * @param idx The output primary index. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t +bw_transform(const sauchar_t *T, sauchar_t *U, + saidx_t *SA /* can NULL */, + saidx_t n, saidx_t *idx); + +/** + * Inverse BW-transforms a given BWTed string. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param A[0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @param idx The primary index. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t +inverse_bw_transform(const sauchar_t *T, sauchar_t *U, + saidx_t *A /* can NULL */, + saidx_t n, saidx_t idx); + +/** + * Checks the correctness of a given suffix array. + * @param T[0..n-1] The input string. + * @param SA[0..n-1] The input suffix array. + * @param n The length of the given string. + * @param verbose The verbose mode. + * @return 0 if no error occurred. + */ +DIVSUFSORT_API +saint_t +sufcheck(const sauchar_t *T, const saidx_t *SA, saidx_t n, saint_t verbose); + +/** + * Search for the pattern P in the string T. + * @param T[0..Tsize-1] The input string. + * @param Tsize The length of the given string. + * @param P[0..Psize-1] The input pattern string. + * @param Psize The length of the given pattern string. + * @param SA[0..SAsize-1] The input suffix array. + * @param SAsize The length of the given suffix array. + * @param idx The output index. + * @return The count of matches if no error occurred, -1 otherwise. + */ +DIVSUFSORT_API +saidx_t +sa_search(const sauchar_t *T, saidx_t Tsize, + const sauchar_t *P, saidx_t Psize, + const saidx_t *SA, saidx_t SAsize, + saidx_t *left); + +/** + * Search for the character c in the string T. + * @param T[0..Tsize-1] The input string. + * @param Tsize The length of the given string. + * @param SA[0..SAsize-1] The input suffix array. + * @param SAsize The length of the given suffix array. + * @param c The input character. + * @param idx The output index. + * @return The count of matches if no error occurred, -1 otherwise. + */ +DIVSUFSORT_API +saidx_t +sa_simplesearch(const sauchar_t *T, saidx_t Tsize, + const saidx_t *SA, saidx_t SAsize, + saint_t c, saidx_t *left); +#endif + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT_H */ diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort.h.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort.h.cmake new file mode 100644 index 0000000..bcaba7c --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort.h.cmake @@ -0,0 +1,180 @@ +/* + * divsufsort@W64BIT@.h for libdivsufsort@W64BIT@ + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT@W64BIT@_H +#define _DIVSUFSORT@W64BIT@_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +@INCFILE@ + +#ifndef DIVSUFSORT_API +# ifdef DIVSUFSORT_BUILD_DLL +# define DIVSUFSORT_API @DIVSUFSORT_EXPORT@ +# else +# define DIVSUFSORT_API @DIVSUFSORT_IMPORT@ +# endif +#endif + +/*- Datatypes -*/ +#ifndef SAUCHAR_T +#define SAUCHAR_T +typedef @SAUCHAR_TYPE@ sauchar_t; +#endif /* SAUCHAR_T */ +#ifndef SAINT_T +#define SAINT_T +typedef @SAINT32_TYPE@ saint_t; +#endif /* SAINT_T */ +#ifndef SAIDX@W64BIT@_T +#define SAIDX@W64BIT@_T +typedef @SAINDEX_TYPE@ saidx@W64BIT@_t; +#endif /* SAIDX@W64BIT@_T */ +#ifndef PRIdSAINT_T +#define PRIdSAINT_T @SAINT_PRId@ +#endif /* PRIdSAINT_T */ +#ifndef PRIdSAIDX@W64BIT@_T +#define PRIdSAIDX@W64BIT@_T @SAINDEX_PRId@ +#endif /* PRIdSAIDX@W64BIT@_T */ + + +/*- Prototypes -*/ + +/** + * Constructs the suffix array of a given string. + * @param T[0..n-1] The input string. + * @param SA[0..n-1] The output array of suffixes. + * @param n The length of the given string. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t +divsufsort@W64BIT@(const sauchar_t *T, saidx@W64BIT@_t *SA, saidx@W64BIT@_t n); + +/** + * Constructs the burrows-wheeler transformed string of a given string. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param A[0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saidx@W64BIT@_t +divbwt@W64BIT@(const sauchar_t *T, sauchar_t *U, saidx@W64BIT@_t *A, saidx@W64BIT@_t n); + +/** + * Returns the version of the divsufsort library. + * @return The version number string. + */ +DIVSUFSORT_API +const char * +divsufsort@W64BIT@_version(void); + + +/** + * Constructs the burrows-wheeler transformed string of a given string and suffix array. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param SA[0..n-1] The suffix array. (can be NULL) + * @param n The length of the given string. + * @param idx The output primary index. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t +bw_transform@W64BIT@(const sauchar_t *T, sauchar_t *U, + saidx@W64BIT@_t *SA /* can NULL */, + saidx@W64BIT@_t n, saidx@W64BIT@_t *idx); + +/** + * Inverse BW-transforms a given BWTed string. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param A[0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @param idx The primary index. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t +inverse_bw_transform@W64BIT@(const sauchar_t *T, sauchar_t *U, + saidx@W64BIT@_t *A /* can NULL */, + saidx@W64BIT@_t n, saidx@W64BIT@_t idx); + +/** + * Checks the correctness of a given suffix array. + * @param T[0..n-1] The input string. + * @param SA[0..n-1] The input suffix array. + * @param n The length of the given string. + * @param verbose The verbose mode. + * @return 0 if no error occurred. + */ +DIVSUFSORT_API +saint_t +sufcheck@W64BIT@(const sauchar_t *T, const saidx@W64BIT@_t *SA, saidx@W64BIT@_t n, saint_t verbose); + +/** + * Search for the pattern P in the string T. + * @param T[0..Tsize-1] The input string. + * @param Tsize The length of the given string. + * @param P[0..Psize-1] The input pattern string. + * @param Psize The length of the given pattern string. + * @param SA[0..SAsize-1] The input suffix array. + * @param SAsize The length of the given suffix array. + * @param idx The output index. + * @return The count of matches if no error occurred, -1 otherwise. + */ +DIVSUFSORT_API +saidx@W64BIT@_t +sa_search@W64BIT@(const sauchar_t *T, saidx@W64BIT@_t Tsize, + const sauchar_t *P, saidx@W64BIT@_t Psize, + const saidx@W64BIT@_t *SA, saidx@W64BIT@_t SAsize, + saidx@W64BIT@_t *left); + +/** + * Search for the character c in the string T. + * @param T[0..Tsize-1] The input string. + * @param Tsize The length of the given string. + * @param SA[0..SAsize-1] The input suffix array. + * @param SAsize The length of the given suffix array. + * @param c The input character. + * @param idx The output index. + * @return The count of matches if no error occurred, -1 otherwise. + */ +DIVSUFSORT_API +saidx@W64BIT@_t +sa_simplesearch@W64BIT@(const sauchar_t *T, saidx@W64BIT@_t Tsize, + const saidx@W64BIT@_t *SA, saidx@W64BIT@_t SAsize, + saint_t c, saidx@W64BIT@_t *left); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT@W64BIT@_H */ diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort_config.h b/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort_config.h new file mode 100644 index 0000000..4054a8a --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort_config.h @@ -0,0 +1,9 @@ +#define HAVE_STRING_H 1 +#define HAVE_STDLIB_H 1 +#define HAVE_MEMORY_H 1 +#define HAVE_STDINT_H 1 +#define INLINE inline + +#ifdef _MSC_VER +#pragma warning( disable : 4244 ) +#endif /* _MSC_VER */ diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort_private.h b/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort_private.h new file mode 100644 index 0000000..b4d97ad --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/include/divsufsort_private.h @@ -0,0 +1,205 @@ +/* + * divsufsort_private.h for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT_PRIVATE_H +#define _DIVSUFSORT_PRIVATE_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#include "divsufsort_config.h" +#include +#include +#if HAVE_STRING_H +# include +#endif +#if HAVE_STDLIB_H +# include +#endif +#if HAVE_MEMORY_H +# include +#endif +#if HAVE_STDDEF_H +# include +#endif +#if HAVE_STRINGS_H +# include +#endif +#if HAVE_INTTYPES_H +# include +#else +# if HAVE_STDINT_H +# include +# endif +#endif +#if defined(BUILD_DIVSUFSORT64) +# include "divsufsort64.h" +# ifndef SAIDX_T +# define SAIDX_T +# define saidx_t saidx64_t +# endif /* SAIDX_T */ +# ifndef PRIdSAIDX_T +# define PRIdSAIDX_T PRIdSAIDX64_T +# endif /* PRIdSAIDX_T */ +# define divsufsort divsufsort64 +# define divbwt divbwt64 +# define divsufsort_version divsufsort64_version +# define bw_transform bw_transform64 +# define inverse_bw_transform inverse_bw_transform64 +# define sufcheck sufcheck64 +# define sa_search sa_search64 +# define sa_simplesearch sa_simplesearch64 +# define sssort sssort64 +# define trsort trsort64 +#else +# include "divsufsort.h" +#endif + + +/*- Constants -*/ +#if !defined(UINT8_MAX) +# define UINT8_MAX (255) +#endif /* UINT8_MAX */ +#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1) +# undef ALPHABET_SIZE +#endif +#if !defined(ALPHABET_SIZE) +# define ALPHABET_SIZE (UINT8_MAX + 1) +#endif +/* for divsufsort.c */ +#define BUCKET_A_SIZE (ALPHABET_SIZE) +#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE) +/* for sssort.c */ +#if defined(SS_INSERTIONSORT_THRESHOLD) +# if SS_INSERTIONSORT_THRESHOLD < 1 +# undef SS_INSERTIONSORT_THRESHOLD +# define SS_INSERTIONSORT_THRESHOLD (1) +# endif +#else +# define SS_INSERTIONSORT_THRESHOLD (8) +#endif +#if defined(SS_BLOCKSIZE) +# if SS_BLOCKSIZE < 0 +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (0) +# elif 32768 <= SS_BLOCKSIZE +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (32767) +# endif +#else +# define SS_BLOCKSIZE (1024) +#endif +/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */ +#if SS_BLOCKSIZE == 0 +# if defined(BUILD_DIVSUFSORT64) +# define SS_MISORT_STACKSIZE (96) +# else +# define SS_MISORT_STACKSIZE (64) +# endif +#elif SS_BLOCKSIZE <= 4096 +# define SS_MISORT_STACKSIZE (16) +#else +# define SS_MISORT_STACKSIZE (24) +#endif +#if defined(BUILD_DIVSUFSORT64) +# define SS_SMERGE_STACKSIZE (64) +#else +# define SS_SMERGE_STACKSIZE (32) +#endif +/* for trsort.c */ +#define TR_INSERTIONSORT_THRESHOLD (8) +#if defined(BUILD_DIVSUFSORT64) +# define TR_STACKSIZE (96) +#else +# define TR_STACKSIZE (64) +#endif + + +/*- Macros -*/ +#ifndef SWAP +# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0) +#endif /* SWAP */ +#ifndef MIN +# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b)) +#endif /* MIN */ +#ifndef MAX +# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b)) +#endif /* MAX */ +#define STACK_PUSH(_a, _b, _c, _d)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize++].d = (_d);\ + } while(0) +#define STACK_PUSH5(_a, _b, _c, _d, _e)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\ + } while(0) +#define STACK_POP(_a, _b, _c, _d)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d;\ + } while(0) +#define STACK_POP5(_a, _b, _c, _d, _e)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\ + } while(0) +/* for divsufsort.c */ +#define BUCKET_A(_c0) bucket_A[(_c0)] +#if ALPHABET_SIZE == 256 +#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)]) +#else +#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)]) +#endif + + +/*- Private Prototypes -*/ +/* sssort.c */ +void +sssort(const sauchar_t *Td, const saidx_t *PA, + saidx_t *first, saidx_t *last, + saidx_t *buf, saidx_t bufsize, + saidx_t depth, saidx_t n, saint_t lastsuffix); +/* trsort.c */ +void +trsort(saidx_t *ISA, saidx_t *SA, saidx_t n, saidx_t depth); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT_PRIVATE_H */ diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/include/lfs.h.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/include/lfs.h.cmake new file mode 100644 index 0000000..d5b84a8 --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/include/lfs.h.cmake @@ -0,0 +1,56 @@ +/* + * lfs.h for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _LFS_H +#define _LFS_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifndef __STRICT_ANSI__ +# define LFS_OFF_T @LFS_OFF_T@ +# define LFS_FOPEN @LFS_FOPEN@ +# define LFS_FTELL @LFS_FTELL@ +# define LFS_FSEEK @LFS_FSEEK@ +# define LFS_PRId @LFS_PRID@ +#else +# define LFS_OFF_T long +# define LFS_FOPEN fopen +# define LFS_FTELL ftell +# define LFS_FSEEK fseek +# define LFS_PRId "ld" +#endif +#ifndef PRIdOFF_T +# define PRIdOFF_T LFS_PRId +#endif + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _LFS_H */ diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/pkgconfig/CMakeLists.txt b/tools/rasm/lzsa-master/src/libdivsufsort/pkgconfig/CMakeLists.txt new file mode 100644 index 0000000..ee7063c --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/pkgconfig/CMakeLists.txt @@ -0,0 +1,9 @@ +## generate libdivsufsort.pc ## +set(W64BIT "") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libdivsufsort.pc.cmake" "${CMAKE_CURRENT_BINARY_DIR}/libdivsufsort.pc" @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libdivsufsort.pc" DESTINATION ${CMAKE_INSTALL_PKGCONFIGDIR}) +if(BUILD_DIVSUFSORT64) + set(W64BIT "64") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libdivsufsort.pc.cmake" "${CMAKE_CURRENT_BINARY_DIR}/libdivsufsort64.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libdivsufsort64.pc" DESTINATION ${CMAKE_INSTALL_PKGCONFIGDIR}) +endif(BUILD_DIVSUFSORT64) diff --git a/tools/rasm/lzsa-master/src/libdivsufsort/pkgconfig/libdivsufsort.pc.cmake b/tools/rasm/lzsa-master/src/libdivsufsort/pkgconfig/libdivsufsort.pc.cmake new file mode 100644 index 0000000..6419d1e --- /dev/null +++ b/tools/rasm/lzsa-master/src/libdivsufsort/pkgconfig/libdivsufsort.pc.cmake @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=@CMAKE_INSTALL_LIBDIR@ +includedir=@CMAKE_INSTALL_INCLUDEDIR@ + +Name: @PROJECT_NAME@@W64BIT@ +Description: @PROJECT_DESCRIPTION@ +Version: @PROJECT_VERSION_FULL@ +URL: @PROJECT_URL@ +Libs: -L${libdir} -ldivsufsort@W64BIT@ +Cflags: -I${includedir} diff --git a/tools/rasm/lzsa-master/src/lzsa.c b/tools/rasm/lzsa-master/src/lzsa.c new file mode 100755 index 0000000..96a460a --- /dev/null +++ b/tools/rasm/lzsa-master/src/lzsa.c @@ -0,0 +1,1109 @@ +/* + * lzsa.c - command line compression utility for the LZSA format + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#endif +#include "lib.h" + +#define OPT_VERBOSE 1 +#define OPT_RAW 2 +#define OPT_FAVOR_RATIO 4 +#define OPT_RAW_BACKWARD 8 +#define OPT_STATS 16 + +#define TOOL_VERSION "1.3.5" + +/*---------------------------------------------------------------------------*/ + +#ifdef _WIN32 +LARGE_INTEGER hpc_frequency; +BOOL hpc_available = FALSE; +#endif + +static void do_init_time() { +#ifdef _WIN32 + hpc_frequency.QuadPart = 0; + hpc_available = QueryPerformanceFrequency(&hpc_frequency); +#endif +} + +static long long do_get_time() { + long long nTime; + +#ifdef _WIN32 + if (hpc_available) { + LARGE_INTEGER nCurTime; + + /* Use HPC hardware for best precision */ + QueryPerformanceCounter(&nCurTime); + nTime = (long long)(nCurTime.QuadPart * 1000000LL / hpc_frequency.QuadPart); + } + else { + struct _timeb tb; + _ftime(&tb); + + nTime = ((long long)tb.time * 1000LL + (long long)tb.millitm) * 1000LL; + } +#else + struct timeval tm; + gettimeofday(&tm, NULL); + + nTime = (long long)tm.tv_sec * 1000000LL + (long long)tm.tv_usec; +#endif + return nTime; +} + +/*---------------------------------------------------------------------------*/ + +static void compression_progress(long long nOriginalSize, long long nCompressedSize) { + if (nOriginalSize >= 1024 * 1024) { + fprintf(stdout, "\r%lld => %lld (%g %%) \b\b\b\b\b", nOriginalSize, nCompressedSize, (double)(nCompressedSize * 100.0 / nOriginalSize)); + fflush(stdout); + } +} + +static int do_compress(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, const unsigned int nOptions, const int nMinMatchSize, const int nFormatVersion) { + long long nStartTime = 0LL, nEndTime = 0LL; + long long nOriginalSize = 0LL, nCompressedSize = 0LL; + int nCommandCount = 0, nSafeDist = 0; + int nFlags; + lzsa_status_t nStatus; + lzsa_stats stats; + + nFlags = 0; + if (nOptions & OPT_FAVOR_RATIO) + nFlags |= LZSA_FLAG_FAVOR_RATIO; + if (nOptions & OPT_RAW) + nFlags |= LZSA_FLAG_RAW_BLOCK; + if (nOptions & OPT_RAW_BACKWARD) + nFlags |= LZSA_FLAG_RAW_BACKWARD; + + if (nOptions & OPT_VERBOSE) { + nStartTime = do_get_time(); + } + + nStatus = lzsa_compress_file(pszInFilename, pszOutFilename, pszDictionaryFilename, nFlags, nMinMatchSize, nFormatVersion, compression_progress, &nOriginalSize, &nCompressedSize, &nCommandCount, &nSafeDist, &stats); + + if ((nOptions & OPT_VERBOSE)) { + nEndTime = do_get_time(); + } + + switch (nStatus) { + case LZSA_ERROR_SRC: fprintf(stderr, "error reading '%s'\n", pszInFilename); break; + case LZSA_ERROR_DST: fprintf(stderr, "error writing '%s'\n", pszOutFilename); break; + case LZSA_ERROR_DICTIONARY: fprintf(stderr, "error reading dictionary '%s'\n", pszDictionaryFilename); break; + case LZSA_ERROR_MEMORY: fprintf(stderr, "out of memory\n"); break; + case LZSA_ERROR_COMPRESSION: fprintf(stderr, "internal compression error\n"); break; + case LZSA_ERROR_RAW_TOOLARGE: fprintf(stderr, "error: raw blocks can only be used with files <= 64 Kb\n"); break; + case LZSA_ERROR_RAW_UNCOMPRESSED: fprintf(stderr, "error: incompressible data needs to be <= 64 Kb in raw blocks\n"); break; + case LZSA_OK: break; + default: fprintf(stderr, "unknown compression error %d\n", nStatus); break; + } + + if (nStatus) + return 100; + + if ((nOptions & OPT_VERBOSE)) { + double fDelta = ((double)(nEndTime - nStartTime)) / 1000000.0; + double fSpeed = ((double)nOriginalSize / 1048576.0) / fDelta; + fprintf(stdout, "\rCompressed '%s' in %g seconds, %.02g Mb/s, %d tokens (%g bytes/token), %lld into %lld bytes ==> %g %%\n", + pszInFilename, fDelta, fSpeed, nCommandCount, (double)nOriginalSize / (double)nCommandCount, + nOriginalSize, nCompressedSize, (double)(nCompressedSize * 100.0 / nOriginalSize)); + if (nOptions & OPT_RAW) { + fprintf(stdout, "Safe distance: %d (0x%X)\n", nSafeDist, nSafeDist); + } + } + + if (nOptions & OPT_STATS) { + if (stats.literals_divisor > 0) + fprintf(stdout, "Literals: min: %d avg: %d max: %d count: %d\n", stats.min_literals, stats.total_literals / stats.literals_divisor, stats.max_literals, stats.literals_divisor); + else + fprintf(stdout, "Literals: none\n"); + if (stats.match_divisor > 0) { + fprintf(stdout, "Offsets: min: %d avg: %d max: %d reps: %d count: %d\n", stats.min_offset, stats.total_offsets / stats.match_divisor, stats.max_offset, stats.num_rep_offsets, stats.match_divisor); + fprintf(stdout, "Match lens: min: %d avg: %d max: %d count: %d\n", stats.min_match_len, stats.total_match_lens / stats.match_divisor, stats.max_match_len, stats.match_divisor); + } + else { + fprintf(stdout, "Offsets: none\n"); + fprintf(stdout, "Match lens: none\n"); + } + if (stats.rle1_divisor > 0) { + fprintf(stdout, "RLE1 lens: min: %d avg: %d max: %d count: %d\n", stats.min_rle1_len, stats.total_rle1_lens / stats.rle1_divisor, stats.max_rle1_len, stats.rle1_divisor); + } + else { + fprintf(stdout, "RLE1 lens: none\n"); + } + if (stats.rle2_divisor > 0) { + fprintf(stdout, "RLE2 lens: min: %d avg: %d max: %d count: %d\n", stats.min_rle2_len, stats.total_rle2_lens / stats.rle2_divisor, stats.max_rle2_len, stats.rle2_divisor); + } + else { + fprintf(stdout, "RLE2 lens: none\n"); + } + } + return 0; +} + +/*---------------------------------------------------------------------------*/ + +static int do_decompress(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, const unsigned int nOptions, int nFormatVersion) { + long long nStartTime = 0LL, nEndTime = 0LL; + long long nOriginalSize = 0LL, nCompressedSize = 0LL; + lzsa_status_t nStatus; + int nFlags; + + nFlags = 0; + if (nOptions & OPT_RAW) + nFlags |= LZSA_FLAG_RAW_BLOCK; + if (nOptions & OPT_RAW_BACKWARD) + nFlags |= LZSA_FLAG_RAW_BACKWARD; + + if (nOptions & OPT_VERBOSE) { + nStartTime = do_get_time(); + } + + nStatus = lzsa_decompress_file(pszInFilename, pszOutFilename, pszDictionaryFilename, nFlags, nFormatVersion, &nOriginalSize, &nCompressedSize); + + switch (nStatus) { + case LZSA_ERROR_SRC: fprintf(stderr, "error reading '%s'\n", pszInFilename); break; + case LZSA_ERROR_DST: fprintf(stderr, "error writing '%s'\n", pszOutFilename); break; + case LZSA_ERROR_DICTIONARY: fprintf(stderr, "error reading dictionary '%s'\n", pszDictionaryFilename); break; + case LZSA_ERROR_MEMORY: fprintf(stderr, "out of memory\n"); break; + case LZSA_ERROR_DECOMPRESSION: fprintf(stderr, "internal decompression error\n"); break; + case LZSA_ERROR_FORMAT: fprintf(stderr, "invalid magic number or format version in input file\n"); break; + case LZSA_OK: break; + default: fprintf(stderr, "unknown decompression error %d\n", nStatus); break; + } + + if (nStatus) { + fprintf(stderr, "decompression error for '%s'\n", pszInFilename); + return 100; + } + else { + if (nOptions & OPT_VERBOSE) { + nEndTime = do_get_time(); + double fDelta = ((double)(nEndTime - nStartTime)) / 1000000.0; + double fSpeed = ((double)nOriginalSize / 1048576.0) / fDelta; + fprintf(stdout, "Decompressed '%s' in %g seconds, %g Mb/s\n", + pszInFilename, fDelta, fSpeed); + } + + return 0; + } +} + +/*---------------------------------------------------------------------------*/ + +typedef struct { + FILE *f; + void *pCompareDataBuf; + size_t nCompareDataSize; +} compare_stream_t; + +void comparestream_close(lzsa_stream_t *stream) { + if (stream->obj) { + compare_stream_t *pCompareStream = (compare_stream_t *)stream->obj; + if (pCompareStream->pCompareDataBuf) { + free(pCompareStream->pCompareDataBuf); + pCompareStream->pCompareDataBuf = NULL; + } + + fclose(pCompareStream->f); + free(pCompareStream); + + stream->obj = NULL; + stream->read = NULL; + stream->write = NULL; + stream->eof = NULL; + stream->close = NULL; + } +} + +size_t comparestream_read(lzsa_stream_t *stream, void *ptr, size_t size) { + return 0; +} + +size_t comparestream_write(lzsa_stream_t *stream, void *ptr, size_t size) { + compare_stream_t *pCompareStream = (compare_stream_t *)stream->obj; + + if (!pCompareStream->pCompareDataBuf || pCompareStream->nCompareDataSize < size) { + pCompareStream->nCompareDataSize = size; + pCompareStream->pCompareDataBuf = realloc(pCompareStream->pCompareDataBuf, pCompareStream->nCompareDataSize); + if (!pCompareStream->pCompareDataBuf) + return 0; + } + + size_t nReadBytes = fread(pCompareStream->pCompareDataBuf, 1, size, pCompareStream->f); + if (nReadBytes != size) { + return 0; + } + + if (memcmp(ptr, pCompareStream->pCompareDataBuf, size)) { + return 0; + } + + return size; +} + +int comparestream_eof(lzsa_stream_t *stream) { + compare_stream_t *pCompareStream = (compare_stream_t *)stream->obj; + return feof(pCompareStream->f); +} + +int comparestream_open(lzsa_stream_t *stream, const char *pszCompareFilename, const char *pszMode) { + compare_stream_t *pCompareStream; + + pCompareStream = (compare_stream_t*)malloc(sizeof(compare_stream_t)); + if (!pCompareStream) + return -1; + + pCompareStream->pCompareDataBuf = NULL; + pCompareStream->nCompareDataSize = 0; + pCompareStream->f = (FILE*)fopen(pszCompareFilename, pszMode); + + if (pCompareStream->f) { + stream->obj = pCompareStream; + stream->read = comparestream_read; + stream->write = comparestream_write; + stream->eof = comparestream_eof; + stream->close = comparestream_close; + return 0; + } + else { + free(pCompareStream); + return -1; + } +} + +static int do_compare(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, const unsigned int nOptions, int nFormatVersion) { + lzsa_stream_t inStream, compareStream; + long long nStartTime = 0LL, nEndTime = 0LL; + long long nOriginalSize = 0LL; + long long nCompressedSize = 0LL; + void *pDictionaryData = NULL; + int nDictionaryDataSize = 0; + lzsa_status_t nStatus; + int nFlags; + + if (lzsa_filestream_open(&inStream, pszInFilename, "rb") < 0) { + fprintf(stderr, "error opening compressed input file\n"); + return 100; + } + + if (comparestream_open(&compareStream, pszOutFilename, "rb") < 0) { + fprintf(stderr, "error opening original uncompressed file\n"); + inStream.close(&inStream); + return 100; + } + + nStatus = lzsa_dictionary_load(pszDictionaryFilename, &pDictionaryData, &nDictionaryDataSize); + if (nStatus) { + compareStream.close(&compareStream); + inStream.close(&inStream); + fprintf(stderr, "error reading dictionary '%s'\n", pszDictionaryFilename); + return 100; + } + + nFlags = 0; + if (nOptions & OPT_RAW) + nFlags |= LZSA_FLAG_RAW_BLOCK; + if (nOptions & OPT_RAW_BACKWARD) + nFlags |= LZSA_FLAG_RAW_BACKWARD; + + if (nOptions & OPT_VERBOSE) { + nStartTime = do_get_time(); + } + + nStatus = lzsa_decompress_stream(&inStream, &compareStream, pDictionaryData, nDictionaryDataSize, nFlags, nFormatVersion, &nOriginalSize, &nCompressedSize); + + switch (nStatus) { + case LZSA_ERROR_SRC: fprintf(stderr, "error reading '%s'\n", pszInFilename); break; + case LZSA_ERROR_DST: fprintf(stderr, "error comparing compressed file '%s' with original '%s'\n", pszInFilename, pszOutFilename); break; + case LZSA_ERROR_MEMORY: fprintf(stderr, "out of memory\n"); break; + case LZSA_ERROR_DECOMPRESSION: fprintf(stderr, "internal decompression error\n"); break; + case LZSA_ERROR_FORMAT: fprintf(stderr, "invalid magic number or format version in input file\n"); break; + case LZSA_OK: break; + default: fprintf(stderr, "unknown decompression error %d\n", nStatus); break; + } + + lzsa_dictionary_free(&pDictionaryData); + compareStream.close(&compareStream); + inStream.close(&inStream); + + if (nStatus) { + return 100; + } + else { + if (nOptions & OPT_VERBOSE) { + nEndTime = do_get_time(); + double fDelta = ((double)(nEndTime - nStartTime)) / 1000000.0; + double fSpeed = ((double)nOriginalSize / 1048576.0) / fDelta; + fprintf(stdout, "Compared '%s' in %g seconds, %g Mb/s\n", + pszInFilename, fDelta, fSpeed); + } + + return 0; + } +} + +/*---------------------------------------------------------------------------*/ + +static void generate_compressible_data(unsigned char *pBuffer, size_t nBufferSize, int nMinMatchSize, unsigned int nSeed, int nNumLiteralValues, float fMatchProbability) { + size_t nIndex = 0; + int nMatchProbability = (int)(fMatchProbability * 1023.0f); + + srand(nSeed); + + if (nIndex >= nBufferSize) return; + pBuffer[nIndex++] = rand() % nNumLiteralValues; + + while (nIndex < nBufferSize) { + if ((rand() & 1023) >= nMatchProbability) { + size_t nLiteralCount = rand() & 127; + if (nLiteralCount > (nBufferSize - nIndex)) + nLiteralCount = nBufferSize - nIndex; + + while (nLiteralCount--) + pBuffer[nIndex++] = rand() % nNumLiteralValues; + } + else { + size_t nMatchLength = nMinMatchSize + (rand() & 1023); + size_t nMatchOffset; + + if (nMatchLength > (nBufferSize - nIndex)) + nMatchLength = nBufferSize - nIndex; + if (nMatchLength > nIndex) + nMatchLength = nIndex; + + if (nMatchLength < nIndex) + nMatchOffset = rand() % (nIndex - nMatchLength); + else + nMatchOffset = 0; + + while (nMatchLength--) { + pBuffer[nIndex] = pBuffer[nIndex - nMatchOffset]; + nIndex++; + } + } + } +} + +static void xor_data(unsigned char *pBuffer, size_t nBufferSize, unsigned int nSeed, float fXorProbability) { + size_t nIndex = 0; + int nXorProbability = (int)(fXorProbability * 1023.0f); + + srand(nSeed); + + if (nIndex >= nBufferSize) return; + + while (nIndex < nBufferSize) { + if ((rand() & 1023) < nXorProbability) { + pBuffer[nIndex] ^= 0xff; + } + nIndex++; + } +} + +static int do_self_test(const unsigned int nOptions, const int nMinMatchSize, int nFormatVersion) { + unsigned char *pGeneratedData; + unsigned char *pCompressedData; + unsigned char *pTmpCompressedData; + unsigned char *pTmpDecompressedData; + size_t nGeneratedDataSize; + size_t nMaxCompressedDataSize; + unsigned int nSeed = 123; + int nFlags; + int i; + + nFlags = 0; + if (nOptions & OPT_FAVOR_RATIO) + nFlags |= LZSA_FLAG_FAVOR_RATIO; + if (nOptions & OPT_RAW) + nFlags |= LZSA_FLAG_RAW_BLOCK; + if (nOptions & OPT_RAW_BACKWARD) + nFlags |= LZSA_FLAG_RAW_BACKWARD; + + pGeneratedData = (unsigned char*)malloc(4 * BLOCK_SIZE); + if (!pGeneratedData) { + fprintf(stderr, "out of memory, %d bytes needed\n", 4 * BLOCK_SIZE); + return 100; + } + + nMaxCompressedDataSize = lzsa_get_max_compressed_size_inmem(4 * BLOCK_SIZE); + pCompressedData = (unsigned char*)malloc(nMaxCompressedDataSize); + if (!pCompressedData) { + free(pGeneratedData); + pGeneratedData = NULL; + + fprintf(stderr, "out of memory, %zd bytes needed\n", nMaxCompressedDataSize); + return 100; + } + + pTmpCompressedData = (unsigned char*)malloc(nMaxCompressedDataSize); + if (!pTmpCompressedData) { + free(pCompressedData); + pCompressedData = NULL; + free(pGeneratedData); + pGeneratedData = NULL; + + fprintf(stderr, "out of memory, %zd bytes needed\n", nMaxCompressedDataSize); + return 100; + } + + pTmpDecompressedData = (unsigned char*)malloc(4 * BLOCK_SIZE); + if (!pTmpDecompressedData) { + free(pTmpCompressedData); + pTmpCompressedData = NULL; + free(pCompressedData); + pCompressedData = NULL; + free(pGeneratedData); + pGeneratedData = NULL; + + fprintf(stderr, "out of memory, %d bytes needed\n", 4 * BLOCK_SIZE); + return 100; + } + + memset(pGeneratedData, 0, 4 * BLOCK_SIZE); + memset(pCompressedData, 0, nMaxCompressedDataSize); + memset(pTmpCompressedData, 0, nMaxCompressedDataSize); + + /* Test compressing with a too small buffer to do anything, expect to fail cleanly */ + for (i = 0; i < 12; i++) { + generate_compressible_data(pGeneratedData, i, nMinMatchSize, nSeed, 256, 0.5f); + lzsa_compress_inmem(pGeneratedData, pCompressedData, i, i, nFlags, nMinMatchSize, nFormatVersion); + } + + size_t nDataSizeStep = 128; + float fProbabilitySizeStep = 0.0005f; + + for (nGeneratedDataSize = 1024; nGeneratedDataSize <= ((size_t)((nOptions & OPT_RAW) ? BLOCK_SIZE : (4 * BLOCK_SIZE))); nGeneratedDataSize += nDataSizeStep) { + float fMatchProbability; + + fprintf(stdout, "size %zd", nGeneratedDataSize); + for (fMatchProbability = 0; fMatchProbability <= 0.995f; fMatchProbability += fProbabilitySizeStep) { + int nNumLiteralValues[12] = { 1, 2, 3, 15, 30, 56, 96, 137, 178, 191, 255, 256 }; + float fXorProbability; + + fputc('.', stdout); + fflush(stdout); + + for (i = 0; i < 12; i++) { + /* Generate data to compress */ + generate_compressible_data(pGeneratedData, nGeneratedDataSize, nMinMatchSize, nSeed, nNumLiteralValues[i], fMatchProbability); + + /* Try to compress it, expected to succeed */ + size_t nActualCompressedSize = lzsa_compress_inmem(pGeneratedData, pCompressedData, nGeneratedDataSize, lzsa_get_max_compressed_size_inmem(nGeneratedDataSize), + nFlags, nMinMatchSize, nFormatVersion); + if (nActualCompressedSize == -1 || (int)nActualCompressedSize < (lzsa_get_header_size() + lzsa_get_frame_size() + lzsa_get_frame_size() /* footer */)) { + free(pTmpDecompressedData); + pTmpDecompressedData = NULL; + free(pTmpCompressedData); + pTmpCompressedData = NULL; + free(pCompressedData); + pCompressedData = NULL; + free(pGeneratedData); + pGeneratedData = NULL; + + fprintf(stderr, "\nself-test: error compressing size %zd, seed %d, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]); + return 100; + } + + /* Try to decompress it, expected to succeed */ + size_t nActualDecompressedSize; + int nDecFormatVersion = nFormatVersion; + nActualDecompressedSize = lzsa_decompress_inmem(pCompressedData, pTmpDecompressedData, nActualCompressedSize, nGeneratedDataSize, nFlags, &nDecFormatVersion); + if (nActualDecompressedSize == -1) { + free(pTmpDecompressedData); + pTmpDecompressedData = NULL; + free(pTmpCompressedData); + pTmpCompressedData = NULL; + free(pCompressedData); + pCompressedData = NULL; + free(pGeneratedData); + pGeneratedData = NULL; + + fprintf(stderr, "\nself-test: error decompressing size %zd, seed %d, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]); + return 100; + } + + if (memcmp(pGeneratedData, pTmpDecompressedData, nGeneratedDataSize)) { + free(pTmpDecompressedData); + pTmpDecompressedData = NULL; + free(pTmpCompressedData); + pTmpCompressedData = NULL; + free(pCompressedData); + pCompressedData = NULL; + free(pGeneratedData); + pGeneratedData = NULL; + + fprintf(stderr, "\nself-test: error comparing decompressed and original data, size %zd, seed %d, match probability %f, literals range %d\n", nGeneratedDataSize, nSeed, fMatchProbability, nNumLiteralValues[i]); + return 100; + } + + /* Try to decompress corrupted data, expected to fail cleanly, without crashing or corrupting memory outside the output buffer */ + for (fXorProbability = 0.05f; fXorProbability <= 0.5f; fXorProbability += 0.05f) { + memcpy(pTmpCompressedData, pCompressedData, nActualCompressedSize); + xor_data(pTmpCompressedData + lzsa_get_header_size() + lzsa_get_frame_size(), nActualCompressedSize - lzsa_get_header_size() - lzsa_get_frame_size() - lzsa_get_frame_size() /* footer */, nSeed, fXorProbability); + nDecFormatVersion = nFormatVersion; + lzsa_decompress_inmem(pTmpCompressedData, pGeneratedData, nActualCompressedSize, nGeneratedDataSize, nFlags, &nDecFormatVersion); + } + } + + nSeed++; + } + + fputc(10, stdout); + fflush(stdout); + + nDataSizeStep <<= 1; + if (nDataSizeStep > (128 * 4096)) + nDataSizeStep = 128 * 4096; + fProbabilitySizeStep *= 1.25; + if (fProbabilitySizeStep > (0.0005f * 4096)) + fProbabilitySizeStep = 0.0005f * 4096; + } + + free(pTmpDecompressedData); + pTmpDecompressedData = NULL; + + free(pTmpCompressedData); + pTmpCompressedData = NULL; + + free(pCompressedData); + pCompressedData = NULL; + + free(pGeneratedData); + pGeneratedData = NULL; + + fprintf(stdout, "All tests passed.\n"); + return 0; +} + +/*---------------------------------------------------------------------------*/ + +static int do_compr_benchmark(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, const unsigned int nOptions, const int nMinMatchSize, int nFormatVersion) { + size_t nFileSize, nMaxCompressedSize; + unsigned char *pFileData; + unsigned char *pCompressedData; + int nFlags; + int i; + + nFlags = 0; + if (nOptions & OPT_FAVOR_RATIO) + nFlags |= LZSA_FLAG_FAVOR_RATIO; + if (nOptions & OPT_RAW) + nFlags |= LZSA_FLAG_RAW_BLOCK; + if (nOptions & OPT_RAW_BACKWARD) + nFlags |= LZSA_FLAG_RAW_BACKWARD; + + if (pszDictionaryFilename) { + fprintf(stderr, "in-memory benchmarking does not support dictionaries\n"); + return 100; + } + + /* Read the whole original file in memory */ + + FILE *f_in = fopen(pszInFilename, "rb"); + if (!f_in) { + fprintf(stderr, "error opening '%s' for reading\n", pszInFilename); + return 100; + } + + fseek(f_in, 0, SEEK_END); + nFileSize = (size_t)ftell(f_in); + fseek(f_in, 0, SEEK_SET); + + pFileData = (unsigned char*)malloc(nFileSize); + if (!pFileData) { + fclose(f_in); + fprintf(stderr, "out of memory for reading '%s', %zd bytes needed\n", pszInFilename, nFileSize); + return 100; + } + + if (fread(pFileData, 1, nFileSize, f_in) != nFileSize) { + free(pFileData); + fclose(f_in); + fprintf(stderr, "I/O error while reading '%s'\n", pszInFilename); + return 100; + } + + fclose(f_in); + + /* Allocate max compressed size */ + + nMaxCompressedSize = lzsa_get_max_compressed_size_inmem(nFileSize); + + pCompressedData = (unsigned char*)malloc(nMaxCompressedSize + 2048); + if (!pCompressedData) { + free(pFileData); + fprintf(stderr, "out of memory for compressing '%s', %zd bytes needed\n", pszInFilename, nMaxCompressedSize); + return 100; + } + + memset(pCompressedData + 1024, 0, nMaxCompressedSize); + + long long nBestCompTime = -1; + + size_t nActualCompressedSize = 0; + size_t nRightGuardPos = nMaxCompressedSize; + + for (i = 0; i < 5; i++) { + unsigned char nGuard = 0x33 + i; + int j; + + /* Write guard bytes around the output buffer, to help check for writes outside of it by the compressor */ + memset(pCompressedData, nGuard, 1024); + memset(pCompressedData + 1024 + nRightGuardPos, nGuard, 1024); + + long long t0 = do_get_time(); + nActualCompressedSize = lzsa_compress_inmem(pFileData, pCompressedData + 1024, nFileSize, nRightGuardPos, nFlags, nMinMatchSize, nFormatVersion); + long long t1 = do_get_time(); + if (nActualCompressedSize == -1) { + free(pCompressedData); + free(pFileData); + fprintf(stderr, "compression error\n"); + return 100; + } + + long long nCurDecTime = t1 - t0; + if (nBestCompTime == -1 || nBestCompTime > nCurDecTime) + nBestCompTime = nCurDecTime; + + /* Check guard bytes before the output buffer */ + for (j = 0; j < 1024; j++) { + if (pCompressedData[j] != nGuard) { + free(pCompressedData); + free(pFileData); + fprintf(stderr, "error, wrote outside of output buffer at %d!\n", j - 1024); + return 100; + } + } + + /* Check guard bytes after the output buffer */ + for (j = 0; j < 1024; j++) { + if (pCompressedData[1024 + nRightGuardPos + j] != nGuard) { + free(pCompressedData); + free(pFileData); + fprintf(stderr, "error, wrote outside of output buffer at %d!\n", j); + return 100; + } + } + + nRightGuardPos = nActualCompressedSize; + } + + if (pszOutFilename) { + FILE *f_out; + + /* Write whole compressed file out */ + + f_out = fopen(pszOutFilename, "wb"); + if (f_out) { + fwrite(pCompressedData + 1024, 1, nActualCompressedSize, f_out); + fclose(f_out); + } + } + + free(pCompressedData); + free(pFileData); + + fprintf(stdout, "compressed size: %zd bytes\n", nActualCompressedSize); + fprintf(stdout, "compression time: %lld microseconds (%g Mb/s)\n", nBestCompTime, ((double)nActualCompressedSize / 1024.0) / ((double)nBestCompTime / 1000.0)); + + return 0; +} + +/*---------------------------------------------------------------------------*/ + +static int do_dec_benchmark(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, const unsigned int nOptions, int nFormatVersion) { + size_t nFileSize, nMaxDecompressedSize; + unsigned char *pFileData; + unsigned char *pDecompressedData; + int nFlags; + int i; + + nFlags = 0; + if (nOptions & OPT_RAW) + nFlags |= LZSA_FLAG_RAW_BLOCK; + if (nOptions & OPT_RAW_BACKWARD) + nFlags |= LZSA_FLAG_RAW_BACKWARD; + + if (pszDictionaryFilename) { + fprintf(stderr, "in-memory benchmarking does not support dictionaries\n"); + return 100; + } + + /* Read the whole compressed file in memory */ + + FILE *f_in = fopen(pszInFilename, "rb"); + if (!f_in) { + fprintf(stderr, "error opening '%s' for reading\n", pszInFilename); + return 100; + } + + fseek(f_in, 0, SEEK_END); + nFileSize = (size_t)ftell(f_in); + fseek(f_in, 0, SEEK_SET); + + pFileData = (unsigned char*)malloc(nFileSize); + if (!pFileData) { + fclose(f_in); + fprintf(stderr, "out of memory for reading '%s', %zd bytes needed\n", pszInFilename, nFileSize); + return 100; + } + + if (fread(pFileData, 1, nFileSize, f_in) != nFileSize) { + free(pFileData); + fclose(f_in); + fprintf(stderr, "I/O error while reading '%s'\n", pszInFilename); + return 100; + } + + fclose(f_in); + + /* Allocate max decompressed size */ + + if (nOptions & OPT_RAW) + nMaxDecompressedSize = 65536; + else + nMaxDecompressedSize = lzsa_get_max_decompressed_size_inmem(pFileData, nFileSize); + if (nMaxDecompressedSize == -1) { + free(pFileData); + fprintf(stderr, "invalid compressed format for file '%s'\n", pszInFilename); + return 100; + } + + pDecompressedData = (unsigned char*)malloc(nMaxDecompressedSize); + if (!pDecompressedData) { + free(pFileData); + fprintf(stderr, "out of memory for decompressing '%s', %zd bytes needed\n", pszInFilename, nMaxDecompressedSize); + return 100; + } + + memset(pDecompressedData, 0, nMaxDecompressedSize); + + long long nBestDecTime = -1; + + size_t nActualDecompressedSize = 0; + for (i = 0; i < 50; i++) { + long long t0 = do_get_time(); + nActualDecompressedSize = lzsa_decompress_inmem(pFileData, pDecompressedData, nFileSize, nMaxDecompressedSize, nFlags, &nFormatVersion); + long long t1 = do_get_time(); + if (nActualDecompressedSize == -1) { + free(pDecompressedData); + free(pFileData); + fprintf(stderr, "decompression error\n"); + return 100; + } + + long long nCurDecTime = t1 - t0; + if (nBestDecTime == -1 || nBestDecTime > nCurDecTime) + nBestDecTime = nCurDecTime; + } + + if (pszOutFilename) { + FILE *f_out; + + /* Write whole decompressed file out */ + + f_out = fopen(pszOutFilename, "wb"); + if (f_out) { + fwrite(pDecompressedData, 1, nActualDecompressedSize, f_out); + fclose(f_out); + } + } + + free(pDecompressedData); + free(pFileData); + + fprintf(stdout, "format: LZSA%d\n", nFormatVersion); + fprintf(stdout, "decompressed size: %zd bytes\n", nActualDecompressedSize); + fprintf(stdout, "decompression time: %lld microseconds (%g Mb/s)\n", nBestDecTime, ((double)nActualDecompressedSize / 1024.0) / ((double)nBestDecTime / 1000.0)); + + return 0; +} + +/*---------------------------------------------------------------------------*/ + +int main(int argc, char **argv) { + int i; + const char *pszInFilename = NULL; + const char *pszOutFilename = NULL; + const char *pszDictionaryFilename = NULL; + int nArgsError = 0; + int nCommandDefined = 0; + int nVerifyCompression = 0; + int nMinMatchDefined = 0; + int nFormatVersionDefined = 0; + char cCommand = 'z'; + int nMinMatchSize = 0; + unsigned int nOptions = OPT_FAVOR_RATIO; + int nFormatVersion = 1; + + for (i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-d")) { + if (!nCommandDefined) { + nCommandDefined = 1; + cCommand = 'd'; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-z")) { + if (!nCommandDefined) { + nCommandDefined = 1; + cCommand = 'z'; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-c")) { + if (!nVerifyCompression) { + nVerifyCompression = 1; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-cbench")) { + if (!nCommandDefined) { + nCommandDefined = 1; + cCommand = 'B'; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-dbench")) { + if (!nCommandDefined) { + nCommandDefined = 1; + cCommand = 'b'; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-test")) { + if (!nCommandDefined) { + nCommandDefined = 1; + cCommand = 't'; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-D")) { + if (!pszDictionaryFilename && (i + 1) < argc) { + pszDictionaryFilename = argv[i + 1]; + i++; + } + else + nArgsError = 1; + } + else if (!strncmp(argv[i], "-D", 2)) { + if (!pszDictionaryFilename) { + pszDictionaryFilename = argv[i] + 2; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-m")) { + if (!nMinMatchDefined && (i + 1) < argc) { + char *pEnd = NULL; + nMinMatchSize = (int)strtol(argv[i + 1], &pEnd, 10); + if (pEnd && pEnd != argv[i + 1] && (nMinMatchSize >= 2 && nMinMatchSize <= 5)) { + i++; + nMinMatchDefined = 1; + nOptions &= (~OPT_FAVOR_RATIO); + } + else { + nArgsError = 1; + } + } + else + nArgsError = 1; + } + else if (!strncmp(argv[i], "-m", 2)) { + if (!nMinMatchDefined) { + char *pEnd = NULL; + nMinMatchSize = (int)strtol(argv[i] + 2, &pEnd, 10); + if (pEnd && pEnd != (argv[i]+2) && (nMinMatchSize >= 2 && nMinMatchSize <= 5)) { + nMinMatchDefined = 1; + nOptions &= (~OPT_FAVOR_RATIO); + } + else { + nArgsError = 1; + } + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "--prefer-ratio")) { + if (!nMinMatchDefined) { + nMinMatchSize = 0; + nMinMatchDefined = 1; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "--prefer-speed")) { + if (!nMinMatchDefined) { + nMinMatchSize = 3; + nOptions &= (~OPT_FAVOR_RATIO); + nMinMatchDefined = 1; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-f")) { + if (!nFormatVersionDefined && (i + 1) < argc) { + char *pEnd = NULL; + nFormatVersion = (int)strtol(argv[i + 1], &pEnd, 10); + if (pEnd && pEnd != argv[i + 1] && (nFormatVersion >= 1 && nFormatVersion <= 2)) { + i++; + nFormatVersionDefined = 1; + } + else { + nArgsError = 1; + } + } + else + nArgsError = 1; + } + else if (!strncmp(argv[i], "-f", 2)) { + if (!nFormatVersionDefined) { + char *pEnd = NULL; + nFormatVersion = (int)strtol(argv[i] + 2, &pEnd, 10); + if (pEnd && pEnd != (argv[i] + 2) && (nFormatVersion >= 1 && nFormatVersion <= 2)) { + nFormatVersionDefined = 1; + } + else { + nArgsError = 1; + } + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-v")) { + if ((nOptions & OPT_VERBOSE) == 0) { + nOptions |= OPT_VERBOSE; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-r")) { + if ((nOptions & OPT_RAW) == 0) { + nOptions |= OPT_RAW; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-b")) { + if ((nOptions & OPT_RAW_BACKWARD) == 0) { + nOptions |= OPT_RAW_BACKWARD; + } + else + nArgsError = 1; + } + else if (!strcmp(argv[i], "-stats")) { + if ((nOptions & OPT_STATS) == 0) { + nOptions |= OPT_STATS; + } + else + nArgsError = 1; + } + else { + if (!pszInFilename) + pszInFilename = argv[i]; + else { + if (!pszOutFilename) + pszOutFilename = argv[i]; + else + nArgsError = 1; + } + } + } + + if (!nArgsError && (nOptions & OPT_RAW_BACKWARD) && !(nOptions & OPT_RAW)) { + fprintf(stderr, "error: -b (compress backwards) requires -r (raw block format)\n"); + return 100; + } + + if (!nArgsError && cCommand == 't') { + return do_self_test(nOptions, nMinMatchSize, nFormatVersion); + } + + if (nArgsError || !pszInFilename || !pszOutFilename) { + fprintf(stderr, "lzsa command-line tool v" TOOL_VERSION " by Emmanuel Marty and spke\n"); + fprintf(stderr, "usage: %s [-c] [-d] [-v] [-r] \n", argv[0]); + fprintf(stderr, " -c: check resulting stream after compressing\n"); + fprintf(stderr, " -d: decompress (default: compress)\n"); + fprintf(stderr, " -cbench: benchmark in-memory compression\n"); + fprintf(stderr, " -dbench: benchmark in-memory decompression\n"); + fprintf(stderr, " -test: run automated self-tests\n"); + fprintf(stderr, " -stats: show compressed data stats\n"); + fprintf(stderr, " -v: be verbose\n"); + fprintf(stderr, " -f : LZSA compression format (1-2)\n"); + fprintf(stderr, " -r: raw block format (max. 64 Kb files)\n"); + fprintf(stderr, " -b: compress backward (requires -r and a backward decompressor)\n"); + fprintf(stderr, " -D : use dictionary file\n"); + fprintf(stderr, " -m : minimum match size (3-5) (default: 3)\n"); + fprintf(stderr, " --prefer-ratio: favor compression ratio (default)\n"); + fprintf(stderr, " --prefer-speed: favor decompression speed (same as -m3)\n"); + return 100; + } + + do_init_time(); + + if (cCommand == 'z') { + int nResult = do_compress(pszInFilename, pszOutFilename, pszDictionaryFilename, nOptions, nMinMatchSize, nFormatVersion); + if (nResult == 0 && nVerifyCompression) { + return do_compare(pszOutFilename, pszInFilename, pszDictionaryFilename, nOptions, nFormatVersion); + } else { + return nResult; + } + } + else if (cCommand == 'd') { + return do_decompress(pszInFilename, pszOutFilename, pszDictionaryFilename, nOptions, nFormatVersion); + } + else if (cCommand == 'B') { + return do_compr_benchmark(pszInFilename, pszOutFilename, pszDictionaryFilename, nOptions, nMinMatchSize, nFormatVersion); + } + else if (cCommand == 'b') { + return do_dec_benchmark(pszInFilename, pszOutFilename, pszDictionaryFilename, nOptions, nFormatVersion); + } + else { + return 100; + } +} diff --git a/tools/rasm/lzsa-master/src/matchfinder.c b/tools/rasm/lzsa-master/src/matchfinder.c new file mode 100644 index 0000000..3de2cfa --- /dev/null +++ b/tools/rasm/lzsa-master/src/matchfinder.c @@ -0,0 +1,361 @@ +/* + * matchfinder.c - LZ match finder implementation + * + * The following copying information applies to this specific source code file: + * + * Written in 2019 by Emmanuel Marty + * Portions written in 2014-2015 by Eric Biggers + * + * To the extent possible under law, the author(s) have dedicated all copyright + * and related and neighboring rights to this software to the public domain + * worldwide via the Creative Commons Zero 1.0 Universal Public Domain + * Dedication (the "CC0"). + * + * This software is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the CC0 for more details. + * + * You should have received a copy of the CC0 along with this software; if not + * see . + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "matchfinder.h" +#include "format.h" +#include "lib.h" + +/** + * Hash index into TAG_BITS + * + * @param nIndex index value + * + * @return hash + */ +static inline int lzsa_get_index_tag(unsigned int nIndex) { + return (int)(((unsigned long long)nIndex * 11400714819323198485ULL) >> (64ULL - TAG_BITS)); +} + +/** + * Parse input data, build suffix array and overlaid data structures to speed up match finding + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nInWindowSize total input size in bytes (previously compressed bytes + bytes to compress) + * + * @return 0 for success, non-zero for failure + */ +int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nInWindowSize) { + unsigned int *intervals = pCompressor->intervals; + + /* Build suffix array from input data */ + if (divsufsort_build_array(&pCompressor->divsufsort_context, pInWindow, (saidx_t*)intervals, nInWindowSize) != 0) { + return 100; + } + + int *PLCP = (int*)pCompressor->pos_data; /* Use temporarily */ + int *Phi = PLCP; + int nCurLen = 0; + int i, r; + + /* Compute the permuted LCP first (Kärkkäinen method) */ + Phi[intervals[0]] = -1; + for (i = 1; i < nInWindowSize; i++) + Phi[intervals[i]] = intervals[i - 1]; + for (i = 0; i < nInWindowSize; i++) { + if (Phi[i] == -1) { + PLCP[i] = 0; + continue; + } + int nMaxLen = (i > Phi[i]) ? (nInWindowSize - i) : (nInWindowSize - Phi[i]); + while (nCurLen < nMaxLen && pInWindow[i + nCurLen] == pInWindow[Phi[i] + nCurLen]) nCurLen++; + PLCP[i] = nCurLen; + if (nCurLen > 0) + nCurLen--; + } + + /* Rotate permuted LCP into the LCP. This has better cache locality than the direct Kasai LCP method. This also + * saves us from having to build the inverse suffix array index, as the LCP is calculated without it using this method, + * and the interval builder below doesn't need it either. */ + intervals[0] &= POS_MASK; + int nMinMatchSize = pCompressor->min_match_size; + + if (pCompressor->format_version >= 2) { + for (i = 1; i < nInWindowSize; i++) { + int nIndex = (int)(intervals[i] & POS_MASK); + int nLen = PLCP[nIndex]; + if (nLen < nMinMatchSize) + nLen = 0; + if (nLen > LCP_MAX) + nLen = LCP_MAX; + int nTaggedLen = 0; + if (nLen) + nTaggedLen = (nLen << TAG_BITS) | (lzsa_get_index_tag((unsigned int)nIndex) & ((1 << TAG_BITS) - 1)); + intervals[i] = ((unsigned int)nIndex) | (((unsigned int)nTaggedLen) << LCP_SHIFT); + } + } + else { + for (i = 1; i < nInWindowSize; i++) { + int nIndex = (int)(intervals[i] & POS_MASK); + int nLen = PLCP[nIndex]; + if (nLen < nMinMatchSize) + nLen = 0; + if (nLen > LCP_AND_TAG_MAX) + nLen = LCP_AND_TAG_MAX; + intervals[i] = ((unsigned int)nIndex) | (((unsigned int)nLen) << LCP_SHIFT); + } + } + + /** + * Build intervals for finding matches + * + * Methodology and code fragment taken from wimlib (CC0 license): + * https://wimlib.net/git/?p=wimlib;a=blob_plain;f=src/lcpit_matchfinder.c;h=a2d6a1e0cd95200d1f3a5464d8359d5736b14cbe;hb=HEAD + */ + unsigned int * const SA_and_LCP = intervals; + unsigned int *pos_data = pCompressor->pos_data; + unsigned int next_interval_idx; + unsigned int *top = pCompressor->open_intervals; + unsigned int prev_pos = SA_and_LCP[0] & POS_MASK; + + *top = 0; + intervals[0] = 0; + next_interval_idx = 1; + + for (r = 1; r < nInWindowSize; r++) { + const unsigned int next_pos = SA_and_LCP[r] & POS_MASK; + const unsigned int next_lcp = SA_and_LCP[r] & LCP_MASK; + const unsigned int top_lcp = *top & LCP_MASK; + + if (next_lcp == top_lcp) { + /* Continuing the deepest open interval */ + pos_data[prev_pos] = *top; + } + else if (next_lcp > top_lcp) { + /* Opening a new interval */ + *++top = next_lcp | next_interval_idx++; + pos_data[prev_pos] = *top; + } + else { + /* Closing the deepest open interval */ + pos_data[prev_pos] = *top; + for (;;) { + const unsigned int closed_interval_idx = *top-- & POS_MASK; + const unsigned int superinterval_lcp = *top & LCP_MASK; + + if (next_lcp == superinterval_lcp) { + /* Continuing the superinterval */ + intervals[closed_interval_idx] = *top; + break; + } + else if (next_lcp > superinterval_lcp) { + /* Creating a new interval that is a + * superinterval of the one being + * closed, but still a subinterval of + * its superinterval */ + *++top = next_lcp | next_interval_idx++; + intervals[closed_interval_idx] = *top; + break; + } + else { + /* Also closing the superinterval */ + intervals[closed_interval_idx] = *top; + } + } + } + prev_pos = next_pos; + } + + /* Close any still-open intervals. */ + pos_data[prev_pos] = *top; + for (; top > pCompressor->open_intervals; top--) + intervals[*top & POS_MASK] = *(top - 1); + + /* Success */ + return 0; +} + +/** + * Find matches at the specified offset in the input window + * + * @param pCompressor compression context + * @param nOffset offset to find matches at, in the input window + * @param pMatches pointer to returned matches + * @param nMaxMatches maximum number of matches to return (0 for none) + * @param nInWindowSize total input size in bytes (previously compressed bytes + bytes to compress) + * + * @return number of matches + */ +int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_match *pMatches, const int nMaxMatches, const int nInWindowSize) { + unsigned int *intervals = pCompressor->intervals; + unsigned int *pos_data = pCompressor->pos_data; + unsigned int ref; + unsigned int super_ref; + unsigned int match_pos; + lzsa_match *matchptr; + int nPrevOffset = 0; + + /** + * Find matches using intervals + * + * Taken from wimlib (CC0 license): + * https://wimlib.net/git/?p=wimlib;a=blob_plain;f=src/lcpit_matchfinder.c;h=a2d6a1e0cd95200d1f3a5464d8359d5736b14cbe;hb=HEAD + */ + + /* Get the deepest lcp-interval containing the current suffix. */ + ref = pos_data[nOffset]; + + pos_data[nOffset] = 0; + + /* Ascend until we reach a visited interval, the root, or a child of the + * root. Link unvisited intervals to the current suffix as we go. */ + while ((super_ref = intervals[ref & POS_MASK]) & LCP_MASK) { + intervals[ref & POS_MASK] = nOffset | VISITED_FLAG; + ref = super_ref; + } + + if (super_ref == 0) { + /* In this case, the current interval may be any of: + * (1) the root; + * (2) an unvisited child of the root */ + + if (ref != 0) /* Not the root? */ + intervals[ref & POS_MASK] = nOffset | VISITED_FLAG; + return 0; + } + + /* Ascend indirectly via pos_data[] links. */ + match_pos = super_ref & EXCL_VISITED_MASK; + matchptr = pMatches; + + if (pCompressor->format_version >= 2 && nInWindowSize < 65536) { + if ((matchptr - pMatches) < nMaxMatches) { + int nMatchOffset = (int)(nOffset - match_pos); + + if (nMatchOffset <= MAX_OFFSET) { + matchptr->length = (unsigned short)(ref >> (LCP_SHIFT + TAG_BITS)); + matchptr->offset = (unsigned short)nMatchOffset; + matchptr++; + + nPrevOffset = nMatchOffset; + } + } + } + + for (;;) { + if ((super_ref = pos_data[match_pos]) > ref) { + match_pos = intervals[super_ref & POS_MASK] & EXCL_VISITED_MASK; + + if (pCompressor->format_version >= 2 && nInWindowSize < 65536) { + if ((matchptr - pMatches) < nMaxMatches) { + int nMatchOffset = (int)(nOffset - match_pos); + + if (nMatchOffset <= MAX_OFFSET && nMatchOffset != nPrevOffset) { + matchptr->length = ((unsigned short)(ref >> (LCP_SHIFT + TAG_BITS))) | 0x8000; + matchptr->offset = (unsigned short)nMatchOffset; + matchptr++; + + nPrevOffset = nMatchOffset; + } + } + } + } + + while ((super_ref = pos_data[match_pos]) > ref) + match_pos = intervals[super_ref & POS_MASK] & EXCL_VISITED_MASK; + intervals[ref & POS_MASK] = nOffset | VISITED_FLAG; + pos_data[match_pos] = ref; + + if ((matchptr - pMatches) < nMaxMatches) { + int nMatchOffset = (int)(nOffset - match_pos); + + if (nMatchOffset <= MAX_OFFSET && nMatchOffset != nPrevOffset) { + if (pCompressor->format_version >= 2) { + matchptr->length = (unsigned short)(ref >> (LCP_SHIFT + TAG_BITS)); + } + else { + matchptr->length = (unsigned short)(ref >> LCP_SHIFT); + } + matchptr->offset = (unsigned short)nMatchOffset; + matchptr++; + } + } + + if (super_ref == 0) + break; + ref = super_ref; + match_pos = intervals[ref & POS_MASK] & EXCL_VISITED_MASK; + + if (pCompressor->format_version >= 2 && nInWindowSize < 65536) { + if ((matchptr - pMatches) < nMaxMatches) { + int nMatchOffset = (int)(nOffset - match_pos); + + if (nMatchOffset <= MAX_OFFSET && nMatchOffset != nPrevOffset) { + matchptr->length = ((unsigned short)(ref >> (LCP_SHIFT + TAG_BITS))) | 0x8000; + matchptr->offset = (unsigned short)nMatchOffset; + + if ((matchptr->length & 0x7fff) > 2) { + matchptr++; + + nPrevOffset = nMatchOffset; + } + } + } + } + } + + return (int)(matchptr - pMatches); +} + +/** + * Skip previously compressed bytes + * + * @param pCompressor compression context + * @param nStartOffset current offset in input window (typically 0) + * @param nEndOffset offset to skip to in input window (typically the number of previously compressed bytes) + */ +void lzsa_skip_matches(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset) { + lzsa_match match; + int i; + + /* Skipping still requires scanning for matches, as this also performs a lazy update of the intervals. However, + * we don't store the matches. */ + for (i = nStartOffset; i < nEndOffset; i++) { + lzsa_find_matches_at(pCompressor, i, &match, 0, 0); + } +} + +/** + * Find all matches for the data to be compressed + * + * @param pCompressor compression context + * @param nMatchesPerOffset maximum number of matches to store for each offset + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + */ +void lzsa_find_all_matches(lzsa_compressor *pCompressor, const int nMatchesPerOffset, const int nStartOffset, const int nEndOffset) { + lzsa_match *pMatch = pCompressor->match; + int i; + + for (i = nStartOffset; i < nEndOffset; i++) { + int nMatches = lzsa_find_matches_at(pCompressor, i, pMatch, nMatchesPerOffset, nEndOffset - nStartOffset); + + while (nMatches < nMatchesPerOffset) { + pMatch[nMatches].length = 0; + pMatch[nMatches].offset = 0; + nMatches++; + } + + pMatch += nMatchesPerOffset; + } +} diff --git a/tools/rasm/lzsa-master/src/matchfinder.h b/tools/rasm/lzsa-master/src/matchfinder.h new file mode 100644 index 0000000..cc3b5ab --- /dev/null +++ b/tools/rasm/lzsa-master/src/matchfinder.h @@ -0,0 +1,91 @@ +/* + * matchfinder.h - LZ match finder definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _MATCHFINDER_H +#define _MATCHFINDER_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Forward declarations */ +typedef struct _lzsa_match lzsa_match; +typedef struct _lzsa_compressor lzsa_compressor; + +/** + * Parse input data, build suffix array and overlaid data structures to speed up match finding + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nInWindowSize total input size in bytes (previously compressed bytes + bytes to compress) + * + * @return 0 for success, non-zero for failure + */ +int lzsa_build_suffix_array(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nInWindowSize); + +/** + * Find matches at the specified offset in the input window + * + * @param pCompressor compression context + * @param nOffset offset to find matches at, in the input window + * @param pMatches pointer to returned matches + * @param nMaxMatches maximum number of matches to return (0 for none) + * @param nInWindowSize total input size in bytes (previously compressed bytes + bytes to compress) + * + * @return number of matches + */ +int lzsa_find_matches_at(lzsa_compressor *pCompressor, const int nOffset, lzsa_match *pMatches, const int nMaxMatches, const int nInWindowSize); + +/** + * Skip previously compressed bytes + * + * @param pCompressor compression context + * @param nStartOffset current offset in input window (typically 0) + * @param nEndOffset offset to skip to in input window (typically the number of previously compressed bytes) + */ +void lzsa_skip_matches(lzsa_compressor *pCompressor, const int nStartOffset, const int nEndOffset); + +/** + * Find all matches for the data to be compressed + * + * @param pCompressor compression context + * @param nMatchesPerOffset maximum number of matches to store for each offset + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + */ +void lzsa_find_all_matches(lzsa_compressor *pCompressor, const int nMatchesPerOffset, const int nStartOffset, const int nEndOffset); + +#ifdef __cplusplus +} +#endif + +#endif /* _MATCHFINDER_H */ diff --git a/tools/rasm/lzsa-master/src/shrink_block_v1.c b/tools/rasm/lzsa-master/src/shrink_block_v1.c new file mode 100644 index 0000000..32c5c38 --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_block_v1.c @@ -0,0 +1,710 @@ +/* + * shrink_block_v1.c - LZSA1 block compressor implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "lib.h" +#include "shrink_block_v1.h" +#include "format.h" + +/** + * Get the number of extra bits required to represent a literals length + * + * @param nLength literals length + * + * @return number of extra bits required + */ +static inline int lzsa_get_literals_varlen_size_v1(const int nLength) { + if (nLength < LITERALS_RUN_LEN_V1) { + return 0; + } + else { + if (nLength < 256) + return 8; + else { + if (nLength < 512) + return 16; + else + return 24; + } + } +} + +/** + * Write extra literals length bytes to output (compressed) buffer. The caller must first check that there is enough + * room to write the bytes. + * + * @param pOutData pointer to output buffer + * @param nOutOffset current write index into output buffer + * @param nLength literals length + */ +static inline int lzsa_write_literals_varlen_v1(unsigned char *pOutData, int nOutOffset, int nLength) { + if (nLength >= LITERALS_RUN_LEN_V1) { + if (nLength < 256) + pOutData[nOutOffset++] = nLength - LITERALS_RUN_LEN_V1; + else { + if (nLength < 512) { + pOutData[nOutOffset++] = 250; + pOutData[nOutOffset++] = nLength - 256; + } + else { + pOutData[nOutOffset++] = 249; + pOutData[nOutOffset++] = nLength & 0xff; + pOutData[nOutOffset++] = (nLength >> 8) & 0xff; + } + } + } + + return nOutOffset; +} + +/** + * Get the number of extra bits required to represent an encoded match length + * + * @param nLength encoded match length (actual match length - MIN_MATCH_SIZE_V1) + * + * @return number of extra bits required + */ +static inline int lzsa_get_match_varlen_size_v1(const int nLength) { + if (nLength < MATCH_RUN_LEN_V1) { + return 0; + } + else { + if ((nLength + MIN_MATCH_SIZE_V1) < 256) + return 8; + else { + if ((nLength + MIN_MATCH_SIZE_V1) < 512) + return 16; + else + return 24; + } + } +} + +/** + * Write extra encoded match length bytes to output (compressed) buffer. The caller must first check that there is enough + * room to write the bytes. + * + * @param pOutData pointer to output buffer + * @param nOutOffset current write index into output buffer + * @param nLength encoded match length (actual match length - MIN_MATCH_SIZE_V1) + */ +static inline int lzsa_write_match_varlen_v1(unsigned char *pOutData, int nOutOffset, int nLength) { + if (nLength >= MATCH_RUN_LEN_V1) { + if ((nLength + MIN_MATCH_SIZE_V1) < 256) + pOutData[nOutOffset++] = nLength - MATCH_RUN_LEN_V1; + else { + if ((nLength + MIN_MATCH_SIZE_V1) < 512) { + pOutData[nOutOffset++] = 239; + pOutData[nOutOffset++] = nLength + MIN_MATCH_SIZE_V1 - 256; + } + else { + pOutData[nOutOffset++] = 238; + pOutData[nOutOffset++] = (nLength + MIN_MATCH_SIZE_V1) & 0xff; + pOutData[nOutOffset++] = ((nLength + MIN_MATCH_SIZE_V1) >> 8) & 0xff; + } + } + } + + return nOutOffset; +} + +/** + * Get offset encoding cost in bits + * + * @param nMatchOffset offset to get cost of + * + * @return cost in bits + */ +static inline int lzsa_get_offset_cost_v1(const unsigned int nMatchOffset) { + return (nMatchOffset <= 256) ? 8 : 16; +} + +/** + * Attempt to pick optimal matches using a forward arrivals parser, so as to produce the smallest possible output that decompresses to the same input + * + * @param pCompressor compression context + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + */ +static void lzsa_optimize_forward_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce) { + lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << ARRIVALS_PER_POSITION_SHIFT); + const int nMinMatchSize = pCompressor->min_match_size; + const int nFavorRatio = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 1 : 0; + const int nModeSwitchPenalty = nFavorRatio ? 0 : MODESWITCH_PENALTY; + const int nDisableScore = nReduce ? 0 : (2 * BLOCK_SIZE); + int i, j, n; + + if ((nEndOffset - nStartOffset) > BLOCK_SIZE) return; + + memset(arrival + (nStartOffset << ARRIVALS_PER_POSITION_SHIFT), 0, sizeof(lzsa_arrival) * ((nEndOffset - nStartOffset + 1) << ARRIVALS_PER_POSITION_SHIFT)); + + arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT].from_slot = -1; + + for (i = nStartOffset; i != nEndOffset; i++) { + lzsa_arrival* cur_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT]; + int m; + + for (j = 0; j < NARRIVALS_PER_POSITION_V1 && cur_arrival[j].from_slot; j++) { + int nPrevCost = cur_arrival[j].cost; + int nCodingChoiceCost = nPrevCost + 8 /* literal */; + int nScore = cur_arrival[j].score + 1; + int nNumLiterals = cur_arrival[j].num_literals + 1; + + if (nNumLiterals == LITERALS_RUN_LEN_V1 || nNumLiterals == 256 || nNumLiterals == 512) { + nCodingChoiceCost += 8; + } + + if (nNumLiterals == 1) + nCodingChoiceCost += nModeSwitchPenalty; + + lzsa_arrival *pDestSlots = &arrival[(i + 1) << ARRIVALS_PER_POSITION_SHIFT]; + for (n = 0; n < NARRIVALS_PER_POSITION_V1 /* we only need the literals + short match cost + long match cost cases */; n++) { + lzsa_arrival *pDestArrival = &pDestSlots[n]; + + if (pDestArrival->from_slot == 0 || + nCodingChoiceCost < pDestArrival->cost || + (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) { + memmove(&arrival[((i + 1) << ARRIVALS_PER_POSITION_SHIFT) + n + 1], + &arrival[((i + 1) << ARRIVALS_PER_POSITION_SHIFT) + n], + sizeof(lzsa_arrival) * (NARRIVALS_PER_POSITION_V1 - n - 1)); + + pDestArrival->cost = nCodingChoiceCost; + pDestArrival->from_pos = i; + pDestArrival->from_slot = j + 1; + pDestArrival->match_len = 0; + pDestArrival->num_literals = nNumLiterals; + pDestArrival->score = nScore; + pDestArrival->rep_offset = cur_arrival[j].rep_offset; + break; + } + } + } + + const lzsa_match *match = pCompressor->match + ((i - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V1); + int nNumArrivalsForThisPos = j; + + for (m = 0; m < NMATCHES_PER_INDEX_V1 && match[m].length; m++) { + int nMatchLen = match[m].length; + int nMatchOffsetCost = lzsa_get_offset_cost_v1(match[m].offset); + int nStartingMatchLen, k; + + if ((i + nMatchLen) > nEndOffset) + nMatchLen = nEndOffset - i; + + if (nMatchLen >= LEAVE_ALONE_MATCH_SIZE) + nStartingMatchLen = nMatchLen; + else + nStartingMatchLen = nMinMatchSize; + for (k = nStartingMatchLen; k <= nMatchLen; k++) { + int nMatchLenCost = lzsa_get_match_varlen_size_v1(k - MIN_MATCH_SIZE_V1); + + lzsa_arrival *pDestSlots = &arrival[(i + k) << ARRIVALS_PER_POSITION_SHIFT]; + + for (j = 0; j < nNumArrivalsForThisPos; j++) { + int nPrevCost = cur_arrival[j].cost; + int nCodingChoiceCost = nPrevCost + 8 /* token */ /* the actual cost of the literals themselves accumulates up the chain */ + nMatchOffsetCost + nMatchLenCost; + int exists = 0; + + if (!cur_arrival[j].num_literals) + nCodingChoiceCost += nModeSwitchPenalty; + + for (n = 0; + n < NARRIVALS_PER_POSITION_V1 && pDestSlots[n].from_slot && pDestSlots[n].cost <= nCodingChoiceCost; + n++) { + if (lzsa_get_offset_cost_v1(pDestSlots[n].rep_offset) == nMatchOffsetCost) { + exists = 1; + break; + } + } + + if (!exists) { + int nScore = cur_arrival[j].score + 5; + + for (n = 0; n < NARRIVALS_PER_POSITION_V1 /* we only need the literals + short match cost + long match cost cases */; n++) { + lzsa_arrival *pDestArrival = &pDestSlots[n]; + + if (pDestArrival->from_slot == 0 || + nCodingChoiceCost < pDestArrival->cost || + (nCodingChoiceCost == pDestArrival->cost && nScore < (pDestArrival->score + nDisableScore))) { + memmove(&pDestSlots[n + 1], + &pDestSlots[n], + sizeof(lzsa_arrival) * (NARRIVALS_PER_POSITION_V1 - n - 1)); + + pDestArrival->cost = nCodingChoiceCost; + pDestArrival->from_pos = i; + pDestArrival->from_slot = j + 1; + pDestArrival->match_len = k; + pDestArrival->num_literals = 0; + pDestArrival->score = nScore; + pDestArrival->rep_offset = match[m].offset; + j = NARRIVALS_PER_POSITION_V1; + break; + } + } + } + } + } + } + } + + lzsa_arrival *end_arrival = &arrival[(i << ARRIVALS_PER_POSITION_SHIFT) + 0]; + + while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0) { + if (end_arrival->from_pos >= nEndOffset) return; + pBestMatch[end_arrival->from_pos].length = end_arrival->match_len; + if (end_arrival->match_len) + pBestMatch[end_arrival->from_pos].offset = end_arrival->rep_offset; + else + pBestMatch[end_arrival->from_pos].offset = 0; + + end_arrival = &arrival[(end_arrival->from_pos << ARRIVALS_PER_POSITION_SHIFT) + (end_arrival->from_slot - 1)]; + } +} + +/** + * Attempt to minimize the number of commands issued in the compressed data block, in order to speed up decompression without + * impacting the compression ratio + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param pBestMatch optimal matches to emit + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * + * @return non-zero if the number of tokens was reduced, 0 if it wasn't + */ +static int lzsa_optimize_command_count_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) { + int i; + int nNumLiterals = 0; + int nDidReduce = 0; + + for (i = nStartOffset; i < nEndOffset; ) { + lzsa_match *pMatch = pBestMatch + i; + + if (pMatch->length == 0 && + (i + 1) < nEndOffset && + pBestMatch[i + 1].length >= MIN_MATCH_SIZE_V1 && + pBestMatch[i + 1].length < MAX_VARLEN && + pBestMatch[i + 1].offset && + i >= pBestMatch[i + 1].offset && + (i + pBestMatch[i + 1].length + 1) <= nEndOffset && + !memcmp(pInWindow + i - (pBestMatch[i + 1].offset), pInWindow + i, pBestMatch[i + 1].length + 1)) { + int nCurLenSize = lzsa_get_match_varlen_size_v1(pBestMatch[i + 1].length - MIN_MATCH_SIZE_V1); + int nReducedLenSize = lzsa_get_match_varlen_size_v1(pBestMatch[i + 1].length + 1 - MIN_MATCH_SIZE_V1); + + if ((nReducedLenSize - nCurLenSize) <= 8) { + /* Merge */ + pBestMatch[i].length = pBestMatch[i + 1].length + 1; + pBestMatch[i].offset = pBestMatch[i + 1].offset; + pBestMatch[i + 1].length = 0; + pBestMatch[i + 1].offset = 0; + nDidReduce = 1; + continue; + } + } + + if (pMatch->length >= MIN_MATCH_SIZE_V1) { + if (pMatch->length <= 9 /* Don't waste time considering large matches, they will always win over literals */ && + (i + pMatch->length) < nEndOffset /* Don't consider the last token in the block, we can only reduce a match inbetween other tokens */) { + int nNextIndex = i + pMatch->length; + int nNextLiterals = 0; + + while (nNextIndex < nEndOffset && pBestMatch[nNextIndex].length < MIN_MATCH_SIZE_V1) { + nNextLiterals++; + nNextIndex++; + } + + /* This command is a match, is followed by 'nNextLiterals' literals and then by another match, or the end of the input. Calculate this command's current cost (excluding 'nNumLiterals' bytes) */ + if ((8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + ((pMatch->offset <= 256) ? 8 : 16) /* match offset */ + lzsa_get_match_varlen_size_v1(pMatch->length - MIN_MATCH_SIZE_V1) + + 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNextLiterals)) >= + (8 /* token */ + (pMatch->length << 3) + lzsa_get_literals_varlen_size_v1(nNumLiterals + pMatch->length + nNextLiterals))) { + /* Reduce */ + int nMatchLen = pMatch->length; + int j; + + for (j = 0; j < nMatchLen; j++) { + pBestMatch[i + j].length = 0; + } + + nDidReduce = 1; + continue; + } + } + + if ((i + pMatch->length) <= nEndOffset && pMatch->offset > 0 && pMatch->length >= MIN_MATCH_SIZE_V1 && + pBestMatch[i + pMatch->length].offset > 0 && + pBestMatch[i + pMatch->length].length >= MIN_MATCH_SIZE_V1 && + (pMatch->length + pBestMatch[i + pMatch->length].length) >= LEAVE_ALONE_MATCH_SIZE && + (pMatch->length + pBestMatch[i + pMatch->length].length) <= MAX_VARLEN && + (i + pMatch->length) > pMatch->offset && + (i + pMatch->length) > pBestMatch[i + pMatch->length].offset && + (i + pMatch->length + pBestMatch[i + pMatch->length].length) <= nEndOffset && + !memcmp(pInWindow + i - pMatch->offset + pMatch->length, + pInWindow + i + pMatch->length - pBestMatch[i + pMatch->length].offset, + pBestMatch[i + pMatch->length].length)) { + + int nCurPartialSize = lzsa_get_match_varlen_size_v1(pMatch->length - MIN_MATCH_SIZE_V1); + nCurPartialSize += 8 /* token */ + lzsa_get_literals_varlen_size_v1(0) + ((pBestMatch[i + pMatch->length].offset <= 256) ? 8 : 16) /* match offset */ + lzsa_get_match_varlen_size_v1(pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V1); + + int nReducedPartialSize = lzsa_get_match_varlen_size_v1(pMatch->length + pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V1); + + if (nCurPartialSize >= nReducedPartialSize) { + int nMatchLen = pMatch->length; + + /* Join */ + + pMatch->length += pBestMatch[i + nMatchLen].length; + pBestMatch[i + nMatchLen].offset = 0; + pBestMatch[i + nMatchLen].length = -1; + continue; + } + } + + i += pMatch->length; + nNumLiterals = 0; + } + else { + nNumLiterals++; + i++; + } + } + + return nDidReduce; +} + +/** + * Get compressed data block size + * + * @param pCompressor compression context + * @param pBestMatch optimal matches to emit + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * + * @return size of compressed data that will be written to output buffer + */ +static int lzsa_get_compressed_size_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) { + int i; + int nNumLiterals = 0; + int nCompressedSize = 0; + + for (i = nStartOffset; i < nEndOffset; ) { + const lzsa_match *pMatch = pBestMatch + i; + + if (pMatch->length >= MIN_MATCH_SIZE_V1) { + int nMatchOffset = pMatch->offset; + int nMatchLen = pMatch->length; + int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V1; + int nTokenLongOffset = (nMatchOffset <= 256) ? 0x00 : 0x80; + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3) + (nTokenLongOffset ? 16 : 8) /* match offset */ + lzsa_get_match_varlen_size_v1(nEncodedMatchLen); + + nCompressedSize += nCommandSize; + nNumLiterals = 0; + i += nMatchLen; + } + else { + nNumLiterals++; + i++; + } + } + + { + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3); + + nCompressedSize += nCommandSize; + nNumLiterals = 0; + } + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + nCompressedSize += 8 * 4; + } + + return nCompressedSize; +} + +/** + * Emit block of compressed data + * + * @param pCompressor compression context + * @param pBestMatch optimal matches to emit + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +static int lzsa_write_block_v1(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) { + int i; + int nNumLiterals = 0; + int nInFirstLiteralOffset = 0; + int nOutOffset = 0; + + for (i = nStartOffset; i < nEndOffset; ) { + const lzsa_match *pMatch = pBestMatch + i; + + if (pMatch->length >= MIN_MATCH_SIZE_V1) { + int nMatchOffset = pMatch->offset; + int nMatchLen = pMatch->length; + int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V1; + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V1) ? LITERALS_RUN_LEN_V1 : nNumLiterals; + int nTokenMatchLen = (nEncodedMatchLen >= MATCH_RUN_LEN_V1) ? MATCH_RUN_LEN_V1 : nEncodedMatchLen; + int nTokenLongOffset = (nMatchOffset <= 256) ? 0x00 : 0x80; + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3) + (nTokenLongOffset ? 16 : 8) /* match offset */ + lzsa_get_match_varlen_size_v1(nEncodedMatchLen); + + if ((nOutOffset + (nCommandSize >> 3)) > nMaxOutDataSize) + return -1; + if (nMatchOffset < MIN_OFFSET || nMatchOffset > MAX_OFFSET) + return -1; + + pOutData[nOutOffset++] = nTokenLongOffset | (nTokenLiteralsLen << 4) | nTokenMatchLen; + nOutOffset = lzsa_write_literals_varlen_v1(pOutData, nOutOffset, nNumLiterals); + + if (nNumLiterals < pCompressor->stats.min_literals || pCompressor->stats.min_literals == -1) + pCompressor->stats.min_literals = nNumLiterals; + if (nNumLiterals > pCompressor->stats.max_literals) + pCompressor->stats.max_literals = nNumLiterals; + pCompressor->stats.total_literals += nNumLiterals; + pCompressor->stats.literals_divisor++; + + if (nNumLiterals != 0) { + memcpy(pOutData + nOutOffset, pInWindow + nInFirstLiteralOffset, nNumLiterals); + nOutOffset += nNumLiterals; + nNumLiterals = 0; + } + + pOutData[nOutOffset++] = (-nMatchOffset) & 0xff; + if (nTokenLongOffset) { + pOutData[nOutOffset++] = (-nMatchOffset) >> 8; + } + nOutOffset = lzsa_write_match_varlen_v1(pOutData, nOutOffset, nEncodedMatchLen); + + if (nMatchOffset < pCompressor->stats.min_offset || pCompressor->stats.min_offset == -1) + pCompressor->stats.min_offset = nMatchOffset; + if (nMatchOffset > pCompressor->stats.max_offset) + pCompressor->stats.max_offset = nMatchOffset; + pCompressor->stats.total_offsets += nMatchOffset; + + if (nMatchLen < pCompressor->stats.min_match_len || pCompressor->stats.min_match_len == -1) + pCompressor->stats.min_match_len = nMatchLen; + if (nMatchLen > pCompressor->stats.max_match_len) + pCompressor->stats.max_match_len = nMatchLen; + pCompressor->stats.total_match_lens += nMatchLen; + pCompressor->stats.match_divisor++; + + if (nMatchOffset == 1) { + if (nMatchLen < pCompressor->stats.min_rle1_len || pCompressor->stats.min_rle1_len == -1) + pCompressor->stats.min_rle1_len = nMatchLen; + if (nMatchLen > pCompressor->stats.max_rle1_len) + pCompressor->stats.max_rle1_len = nMatchLen; + pCompressor->stats.total_rle1_lens += nMatchLen; + pCompressor->stats.rle1_divisor++; + } + else if (nMatchOffset == 2) { + if (nMatchLen < pCompressor->stats.min_rle2_len || pCompressor->stats.min_rle2_len == -1) + pCompressor->stats.min_rle2_len = nMatchLen; + if (nMatchLen > pCompressor->stats.max_rle2_len) + pCompressor->stats.max_rle2_len = nMatchLen; + pCompressor->stats.total_rle2_lens += nMatchLen; + pCompressor->stats.rle2_divisor++; + } + + i += nMatchLen; + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + int nCurSafeDist = (i - nStartOffset) - nOutOffset; + if (nCurSafeDist >= 0 && pCompressor->safe_dist < nCurSafeDist) + pCompressor->safe_dist = nCurSafeDist; + } + + pCompressor->num_commands++; + } + else { + if (nNumLiterals == 0) + nInFirstLiteralOffset = i; + nNumLiterals++; + i++; + } + } + + { + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V1) ? LITERALS_RUN_LEN_V1 : nNumLiterals; + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3); + + if ((nOutOffset + (nCommandSize >> 3)) > nMaxOutDataSize) + return -1; + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) + pOutData[nOutOffset++] = (nTokenLiteralsLen << 4) | 0x0f; + else + pOutData[nOutOffset++] = (nTokenLiteralsLen << 4) | 0x00; + nOutOffset = lzsa_write_literals_varlen_v1(pOutData, nOutOffset, nNumLiterals); + + if (nNumLiterals < pCompressor->stats.min_literals || pCompressor->stats.min_literals == -1) + pCompressor->stats.min_literals = nNumLiterals; + if (nNumLiterals > pCompressor->stats.max_literals) + pCompressor->stats.max_literals = nNumLiterals; + pCompressor->stats.total_literals += nNumLiterals; + pCompressor->stats.literals_divisor++; + + if (nNumLiterals != 0) { + memcpy(pOutData + nOutOffset, pInWindow + nInFirstLiteralOffset, nNumLiterals); + nOutOffset += nNumLiterals; + nNumLiterals = 0; + } + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + int nCurSafeDist = (i - nStartOffset) - nOutOffset; + if (nCurSafeDist >= 0 && pCompressor->safe_dist < nCurSafeDist) + pCompressor->safe_dist = nCurSafeDist; + } + + pCompressor->num_commands++; + } + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + /* Emit EOD marker for raw block */ + + if ((nOutOffset + 4) > nMaxOutDataSize) + return -1; + + pOutData[nOutOffset++] = 0; + pOutData[nOutOffset++] = 238; + pOutData[nOutOffset++] = 0; + pOutData[nOutOffset++] = 0; + } + + return nOutOffset; +} + +/** + * Emit raw block of uncompressible data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +static int lzsa_write_raw_uncompressed_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) { + int nNumLiterals = nEndOffset - nStartOffset; + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V1) ? LITERALS_RUN_LEN_V1 : nNumLiterals; + int nOutOffset = 0; + + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v1(nNumLiterals) + (nNumLiterals << 3) + 4; + if ((nOutOffset + (nCommandSize >> 3)) > nMaxOutDataSize) + return -1; + + pCompressor->num_commands = 0; + pOutData[nOutOffset++] = (nTokenLiteralsLen << 4) | 0x0f; + + nOutOffset = lzsa_write_literals_varlen_v1(pOutData, nOutOffset, nNumLiterals); + + if (nNumLiterals != 0) { + memcpy(pOutData + nOutOffset, pInWindow + nStartOffset, nNumLiterals); + nOutOffset += nNumLiterals; + nNumLiterals = 0; + } + + pCompressor->num_commands++; + + /* Emit EOD marker for raw block */ + + pOutData[nOutOffset++] = 0; + pOutData[nOutOffset++] = 238; + pOutData[nOutOffset++] = 0; + pOutData[nOutOffset++] = 0; + + return nOutOffset; +} + +/** + * Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA1 data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nPreviousBlockSize number of previously compressed bytes (or 0 for none) + * @param nInDataSize number of input bytes to compress + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { + int nResult, nBaseCompressedSize; + + /* Compress optimally without breaking ties in favor of less tokens */ + + memset(pCompressor->best_match, 0, BLOCK_SIZE * sizeof(lzsa_match)); + lzsa_optimize_forward_v1(pCompressor, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 0 /* reduce */); + + int nDidReduce; + int nPasses = 0; + do { + nDidReduce = lzsa_optimize_command_count_v1(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + nPasses++; + } while (nDidReduce && nPasses < 20); + + nBaseCompressedSize = lzsa_get_compressed_size_v1(pCompressor, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + lzsa_match *pBestMatch = pCompressor->best_match - nPreviousBlockSize; + + if (nBaseCompressedSize > 0 && nInDataSize < 65536) { + int nReducedCompressedSize; + + /* Compress optimally and do break ties in favor of less tokens */ + memset(pCompressor->improved_match, 0, BLOCK_SIZE * sizeof(lzsa_match)); + lzsa_optimize_forward_v1(pCompressor, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 1 /* reduce */); + + nPasses = 0; + do { + nDidReduce = lzsa_optimize_command_count_v1(pCompressor, pInWindow, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + nPasses++; + } while (nDidReduce && nPasses < 20); + + nReducedCompressedSize = lzsa_get_compressed_size_v1(pCompressor, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + if (nReducedCompressedSize > 0 && nReducedCompressedSize <= nBaseCompressedSize) { + /* Pick the parse with the reduced number of tokens as it didn't negatively affect the size */ + pBestMatch = pCompressor->improved_match - nPreviousBlockSize; + } + } + + nResult = lzsa_write_block_v1(pCompressor, pBestMatch, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + nResult = lzsa_write_raw_uncompressed_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + } + + return nResult; +} diff --git a/tools/rasm/lzsa-master/src/shrink_block_v1.h b/tools/rasm/lzsa-master/src/shrink_block_v1.h new file mode 100644 index 0000000..cc89cde --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_block_v1.h @@ -0,0 +1,53 @@ +/* + * shrink_block_v1.h - LZSA1 block compressor definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _SHRINK_BLOCK_V1_H +#define _SHRINK_BLOCK_V1_H + +/* Forward declarations */ +typedef struct _lzsa_compressor lzsa_compressor; + +/** + * Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA1 data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nPreviousBlockSize number of previously compressed bytes (or 0 for none) + * @param nInDataSize number of input bytes to compress + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +int lzsa_optimize_and_write_block_v1(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize); + +#endif /* _SHRINK_BLOCK_V1_H */ diff --git a/tools/rasm/lzsa-master/src/shrink_block_v2.c b/tools/rasm/lzsa-master/src/shrink_block_v2.c new file mode 100644 index 0000000..0aba0c0 --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_block_v2.c @@ -0,0 +1,1371 @@ +/* + * shrink_block_v2.c - LZSA2 block compressor implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "lib.h" +#include "shrink_block_v2.h" +#include "format.h" + +/** + * Write 4-bit nibble to output (compressed) buffer + * + * @param pOutData pointer to output buffer + * @param nOutOffset current write index into output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * @param nCurNibbleOffset write index into output buffer, of current byte being filled with nibbles + * @param nNibbleValue value to write (0..15) + */ +static int lzsa_write_nibble_v2(unsigned char *pOutData, int nOutOffset, const int nMaxOutDataSize, int *nCurNibbleOffset, int nNibbleValue) { + if (nOutOffset < 0) return -1; + + if ((*nCurNibbleOffset) == -1) { + if (nOutOffset >= nMaxOutDataSize) return -1; + (*nCurNibbleOffset) = nOutOffset; + pOutData[nOutOffset++] = nNibbleValue << 4; + } + else { + pOutData[*nCurNibbleOffset] = (pOutData[*nCurNibbleOffset]) | (nNibbleValue & 0x0f); + (*nCurNibbleOffset) = -1; + } + + return nOutOffset; +} + +/** + * Get the number of extra bits required to represent a literals length + * + * @param nLength literals length + * + * @return number of extra bits required + */ +static inline int lzsa_get_literals_varlen_size_v2(const int nLength) { + if (nLength < LITERALS_RUN_LEN_V2) { + return 0; + } + else { + if (nLength < (LITERALS_RUN_LEN_V2 + 15)) { + return 4; + } + else { + if (nLength < 256) + return 4+8; + else { + return 4+24; + } + } + } +} + +/** + * Write extra literals length bytes to output (compressed) buffer. The caller must first check that there is enough + * room to write the bytes. + * + * @param pOutData pointer to output buffer + * @param nOutOffset current write index into output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * @param nCurNibbleOffset write index into output buffer, of current byte being filled with nibbles + * @param nLength literals length + */ +static inline int lzsa_write_literals_varlen_v2(unsigned char *pOutData, int nOutOffset, const int nMaxOutDataSize, int *nCurNibbleOffset, int nLength) { + if (nLength >= LITERALS_RUN_LEN_V2) { + if (nLength < (LITERALS_RUN_LEN_V2 + 15)) { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, nCurNibbleOffset, nLength - LITERALS_RUN_LEN_V2); + } + else { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, nCurNibbleOffset, 15); + if (nOutOffset < 0) return -1; + + if (nLength < 256) + pOutData[nOutOffset++] = nLength - 18; + else { + pOutData[nOutOffset++] = 239; + pOutData[nOutOffset++] = nLength & 0xff; + pOutData[nOutOffset++] = (nLength >> 8) & 0xff; + } + } + } + + return nOutOffset; +} + +/** + * Get the number of extra bits required to represent an encoded match length + * + * @param nLength encoded match length (actual match length - MIN_MATCH_SIZE_V2) + * + * @return number of extra bits required + */ +static inline int lzsa_get_match_varlen_size_v2(const int nLength) { + if (nLength < MATCH_RUN_LEN_V2) { + return 0; + } + else { + if (nLength < (MATCH_RUN_LEN_V2 + 15)) + return 4; + else { + if ((nLength + MIN_MATCH_SIZE_V2) < 256) + return 4+8; + else { + return 4 + 24; + } + } + } +} + +/** + * Write extra encoded match length bytes to output (compressed) buffer. The caller must first check that there is enough + * room to write the bytes. + * + * @param pOutData pointer to output buffer + * @param nOutOffset current write index into output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * @param nCurNibbleOffset write index into output buffer, of current byte being filled with nibbles + * @param nLength encoded match length (actual match length - MIN_MATCH_SIZE_V2) + */ +static inline int lzsa_write_match_varlen_v2(unsigned char *pOutData, int nOutOffset, const int nMaxOutDataSize, int *nCurNibbleOffset, int nLength) { + if (nLength >= MATCH_RUN_LEN_V2) { + if (nLength < (MATCH_RUN_LEN_V2 + 15)) { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, nCurNibbleOffset, nLength - MATCH_RUN_LEN_V2); + } + else { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, nCurNibbleOffset, 15); + if (nOutOffset < 0) return -1; + + if ((nLength + MIN_MATCH_SIZE_V2) < 256) + pOutData[nOutOffset++] = nLength + MIN_MATCH_SIZE_V2 - 24; + else { + pOutData[nOutOffset++] = 233; + pOutData[nOutOffset++] = (nLength + MIN_MATCH_SIZE_V2) & 0xff; + pOutData[nOutOffset++] = ((nLength + MIN_MATCH_SIZE_V2) >> 8) & 0xff; + } + } + } + + return nOutOffset; +} + +/** + * Insert forward rep candidate + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param i input data window position whose matches are being considered + * @param nMatchOffset match offset to use as rep candidate + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param nDepth current insertion depth + */ +static void lzsa_insert_forward_match_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int i, const int nMatchOffset, const int nStartOffset, const int nEndOffset, int nDepth) { + lzsa_arrival *arrival = pCompressor->arrival + ((i - nStartOffset) << ARRIVALS_PER_POSITION_SHIFT); + const int *rle_end = (int*)pCompressor->intervals /* reuse */; + lzsa_match* visited = ((lzsa_match*)pCompressor->pos_data) - nStartOffset /* reuse */; + int j; + + for (j = 0; j < NARRIVALS_PER_POSITION_V2_BIG && arrival[j].from_slot; j++) { + int nRepOffset = arrival[j].rep_offset; + + if (nMatchOffset != nRepOffset && nRepOffset && arrival[j].rep_len >= MIN_MATCH_SIZE_V2) { + int nRepPos = arrival[j].rep_pos; + int nRepLen = arrival[j].rep_len; + + if (nRepPos > nMatchOffset && + (nRepPos + nRepLen) <= nEndOffset && + pCompressor->match[((nRepPos - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V2) + NMATCHES_PER_INDEX_V2 - 1].length == 0) { + + if (visited[nRepPos].offset != nMatchOffset || visited[nRepPos].length > nRepLen) { + visited[nRepPos].offset = nMatchOffset; + visited[nRepPos].length = nRepLen; + + if (pInWindow[nRepPos] == pInWindow[nRepPos - nMatchOffset]) { + int nLen0 = rle_end[nRepPos - nMatchOffset] - (nRepPos - nMatchOffset); + int nLen1 = rle_end[nRepPos] - (nRepPos); + int nMinLen = (nLen0 < nLen1) ? nLen0 : nLen1; + + if (nMinLen >= nRepLen || !memcmp(pInWindow + nRepPos + nMinLen, pInWindow + nRepPos + nMinLen - nMatchOffset, nRepLen - nMinLen)) { + visited[nRepPos].length = 0; + + lzsa_match* fwd_match = pCompressor->match + ((nRepPos - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V2); + int r; + + for (r = 0; r < NMATCHES_PER_INDEX_V2 && fwd_match[r].length >= MIN_MATCH_SIZE_V2; r++) { + if (fwd_match[r].offset == nMatchOffset) { + r = NMATCHES_PER_INDEX_V2; + break; + } + } + + if (r < NMATCHES_PER_INDEX_V2) { + int nMaxRepLen = nEndOffset - nRepPos; + if (nMaxRepLen > LCP_MAX) + nMaxRepLen = LCP_MAX; + int nCurRepLen = (nMinLen > nRepLen) ? nMinLen : nRepLen; + if (nCurRepLen > nMaxRepLen) + nCurRepLen = nMaxRepLen; + const unsigned char* pInWindowMax = pInWindow + nRepPos + nMaxRepLen; + const unsigned char* pInWindowAtRepPos = pInWindow + nRepPos + nCurRepLen; + while ((pInWindowAtRepPos + 8) < pInWindowMax && !memcmp(pInWindowAtRepPos, pInWindowAtRepPos - nMatchOffset, 8)) + pInWindowAtRepPos += 8; + while ((pInWindowAtRepPos + 4) < pInWindowMax && !memcmp(pInWindowAtRepPos, pInWindowAtRepPos - nMatchOffset, 4)) + pInWindowAtRepPos += 4; + while (pInWindowAtRepPos < pInWindowMax && pInWindowAtRepPos[0] == pInWindowAtRepPos[-nMatchOffset]) + pInWindowAtRepPos++; + + nCurRepLen = (int)(pInWindowAtRepPos - (pInWindow + nRepPos)); + fwd_match[r].offset = nMatchOffset; + fwd_match[r].length = nCurRepLen; + + if (nDepth < 9) + lzsa_insert_forward_match_v2(pCompressor, pInWindow, nRepPos, nMatchOffset, nStartOffset, nEndOffset, nDepth + 1); + } + } + } + } + } + } + } +} + +/** + * Attempt to pick optimal matches using a forward arrivals parser, so as to produce the smallest possible output that decompresses to the same input + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param pBestMatch pointer to buffer for outputting optimal matches + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param nReduce non-zero to reduce the number of tokens when the path costs are equal, zero not to + * @param nInsertForwardReps non-zero to insert forward repmatch candidates, zero to use the previously inserted candidates + * @param nArrivalsPerPosition number of arrivals to record per input buffer position + */ +static void lzsa_optimize_forward_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset, const int nReduce, const int nInsertForwardReps, const int nArrivalsPerPosition) { + lzsa_arrival *arrival = pCompressor->arrival - (nStartOffset << ARRIVALS_PER_POSITION_SHIFT); + const int *rle_end = (int*)pCompressor->intervals /* reuse */; + lzsa_match *visited = ((lzsa_match*)pCompressor->pos_data) - nStartOffset /* reuse */; + char *nRepLenHandledMask = pCompressor->rep_handled_mask; + const int nModeSwitchPenalty = (pCompressor->flags & LZSA_FLAG_FAVOR_RATIO) ? 0 : MODESWITCH_PENALTY; + const int nMinMatchSize = pCompressor->min_match_size; + const int nDisableScore = nReduce ? 0 : (2 * BLOCK_SIZE); + const int nMaxRepInsertedLen = nReduce ? LEAVE_ALONE_MATCH_SIZE : 0; + const int nLeaveAloneMatchSize = (nArrivalsPerPosition == NARRIVALS_PER_POSITION_V2_SMALL) ? LEAVE_ALONE_MATCH_SIZE_SMALL : LEAVE_ALONE_MATCH_SIZE; + int i, j, n; + + if ((nEndOffset - nStartOffset) > BLOCK_SIZE) return; + + memset(arrival + (nStartOffset << ARRIVALS_PER_POSITION_SHIFT), 0, sizeof(lzsa_arrival) * ((nEndOffset - nStartOffset + 1) << ARRIVALS_PER_POSITION_SHIFT)); + + for (i = (nStartOffset << ARRIVALS_PER_POSITION_SHIFT); i != ((nEndOffset + 1) << ARRIVALS_PER_POSITION_SHIFT); i++) { + arrival[i].cost = 0x40000000; + } + + arrival[nStartOffset << ARRIVALS_PER_POSITION_SHIFT].from_slot = -1; + + if (nInsertForwardReps) { + memset(visited + nStartOffset, 0, (nEndOffset - nStartOffset) * sizeof(lzsa_match)); + } + + for (i = nStartOffset; i != nEndOffset; i++) { + lzsa_arrival *cur_arrival = &arrival[i << ARRIVALS_PER_POSITION_SHIFT]; + int m; + + for (j = 0; j < nArrivalsPerPosition && cur_arrival[j].from_slot; j++) { + const int nPrevCost = cur_arrival[j].cost & 0x3fffffff; + int nCodingChoiceCost = nPrevCost + 8 /* literal */; + int nNumLiterals = cur_arrival[j].num_literals + 1; + + if (nNumLiterals == LITERALS_RUN_LEN_V2) { + nCodingChoiceCost += 4; + } + else if (nNumLiterals == (LITERALS_RUN_LEN_V2 + 15)) { + nCodingChoiceCost += 8; + } + else if (nNumLiterals == 256) { + nCodingChoiceCost += 16; + } + + if (nNumLiterals == 1) + nCodingChoiceCost += nModeSwitchPenalty; + + lzsa_arrival *pDestSlots = &cur_arrival[1 << ARRIVALS_PER_POSITION_SHIFT]; + if (nCodingChoiceCost <= pDestSlots[nArrivalsPerPosition - 1].cost) { + int nRepOffset = cur_arrival[j].rep_offset; + int exists = 0; + + for (n = 0; + n < nArrivalsPerPosition && pDestSlots[n].cost < nCodingChoiceCost; + n++) { + if (pDestSlots[n].rep_offset == nRepOffset) { + exists = 1; + break; + } + } + + if (!exists) { + int nScore = cur_arrival[j].score + 1; + + for (; + n < nArrivalsPerPosition && pDestSlots[n].cost == nCodingChoiceCost && nScore >= (pDestSlots[n].score + nDisableScore); + n++) { + if (pDestSlots[n].rep_offset == nRepOffset) { + exists = 1; + break; + } + } + + if (!exists) { + if (n < nArrivalsPerPosition) { + int nn; + + for (nn = n; + nn < nArrivalsPerPosition && pDestSlots[nn].cost == nCodingChoiceCost; + nn++) { + if (pDestSlots[nn].rep_offset == nRepOffset) { + exists = 1; + break; + } + } + + if (!exists) { + int z; + + for (z = n; z < nArrivalsPerPosition - 1 && pDestSlots[z].from_slot; z++) { + if (pDestSlots[z].rep_offset == nRepOffset) + break; + } + + memmove(&pDestSlots[n + 1], + &pDestSlots[n], + sizeof(lzsa_arrival) * (z - n)); + + lzsa_arrival* pDestArrival = &pDestSlots[n]; + pDestArrival->cost = nCodingChoiceCost; + pDestArrival->from_pos = i; + pDestArrival->from_slot = j + 1; + pDestArrival->match_len = 0; + pDestArrival->num_literals = nNumLiterals; + pDestArrival->score = nScore; + pDestArrival->rep_offset = nRepOffset; + pDestArrival->rep_pos = cur_arrival[j].rep_pos; + pDestArrival->rep_len = cur_arrival[j].rep_len; + } + } + } + } + } + } + + lzsa_match *match = pCompressor->match + ((i - nStartOffset) << MATCHES_PER_INDEX_SHIFT_V2); + int nNumArrivalsForThisPos = j, nMinOverallRepLen = 0, nMaxOverallRepLen = 0; + + int nRepLenForArrival[NARRIVALS_PER_POSITION_V2_BIG]; + memset(nRepLenForArrival, 0, nArrivalsPerPosition * sizeof(int)); + + int nMaxRepLenForPos = nEndOffset - i; + if (nMaxRepLenForPos > LCP_MAX) + nMaxRepLenForPos = LCP_MAX; + const unsigned char* pInWindowStart = pInWindow + i; + const unsigned char* pInWindowMax = pInWindowStart + nMaxRepLenForPos; + + for (j = 0; j < nNumArrivalsForThisPos && (i + MIN_MATCH_SIZE_V2) <= nEndOffset; j++) { + int nRepOffset = cur_arrival[j].rep_offset; + + if (nRepOffset) { + if (i > nRepOffset) { + if (pInWindow[i] == pInWindow[i - nRepOffset]) { + const unsigned char* pInWindowAtPos; + + int nLen0 = rle_end[i - nRepOffset] - (i - nRepOffset); + int nLen1 = rle_end[i] - (i); + int nMinLen = (nLen0 < nLen1) ? nLen0 : nLen1; + + if (nMinLen > nMaxRepLenForPos) + nMinLen = nMaxRepLenForPos; + pInWindowAtPos = pInWindowStart + nMinLen; + + while ((pInWindowAtPos + 8) < pInWindowMax && !memcmp(pInWindowAtPos - nRepOffset, pInWindowAtPos, 8)) + pInWindowAtPos += 8; + while ((pInWindowAtPos + 4) < pInWindowMax && !memcmp(pInWindowAtPos - nRepOffset, pInWindowAtPos, 4)) + pInWindowAtPos += 4; + while (pInWindowAtPos < pInWindowMax && pInWindowAtPos[-nRepOffset] == pInWindowAtPos[0]) + pInWindowAtPos++; + nRepLenForArrival[j] = (int)(pInWindowAtPos - pInWindowStart); + + if (nMaxOverallRepLen < nRepLenForArrival[j]) + nMaxOverallRepLen = nRepLenForArrival[j]; + } + } + } + } + + if (!nReduce) { + memset(nRepLenHandledMask, 0, nArrivalsPerPosition * ((LCP_MAX + 1) / 8) * sizeof(char)); + } + + for (m = 0; m < NMATCHES_PER_INDEX_V2 && match[m].length; m++) { + int nMatchLen = match[m].length & 0x7fff; + int nMatchOffset = match[m].offset; + int nScorePenalty = 3 + ((match[m].length & 0x8000) >> 15); + int nNoRepmatchOffsetCost = (nMatchOffset <= 32) ? 4 : ((nMatchOffset <= 512) ? 8 : ((nMatchOffset <= (8192 + 512)) ? 12 : 16)); + int nStartingMatchLen, k; + + if ((i + nMatchLen) > nEndOffset) + nMatchLen = nEndOffset - i; + + if (nInsertForwardReps) + lzsa_insert_forward_match_v2(pCompressor, pInWindow, i, nMatchOffset, nStartOffset, nEndOffset, 0); + + int nNonRepMatchArrivalIdx = -1; + for (j = 0; j < nNumArrivalsForThisPos; j++) { + int nRepOffset = cur_arrival[j].rep_offset; + + if (nMatchOffset != nRepOffset) { + nNonRepMatchArrivalIdx = j; + break; + } + } + + int nMatchLenCost; + if (nMatchLen >= nLeaveAloneMatchSize) { + nStartingMatchLen = nMatchLen; + nMatchLenCost = 4 + 24 + 8 /* token */; + } + else { + nStartingMatchLen = nMinMatchSize; + nMatchLenCost = 0 + 8 /* token */; + } + + for (k = nStartingMatchLen; k <= nMatchLen; k++) { + if (k == (MATCH_RUN_LEN_V2 + MIN_MATCH_SIZE_V2)) { + nMatchLenCost = 4 + 8 /* token */; + } + else { + if (k == (MATCH_RUN_LEN_V2 + 15 + MIN_MATCH_SIZE_V2)) + nMatchLenCost = 4 + 8 + 8 /* token */; + else { + if (k == 256) + nMatchLenCost = 4 + 24 + 8 /* token */; + } + } + + lzsa_arrival *pDestSlots = &cur_arrival[k << ARRIVALS_PER_POSITION_SHIFT]; + + /* Insert non-repmatch candidate */ + + if (nNonRepMatchArrivalIdx >= 0) { + const int nPrevCost = cur_arrival[nNonRepMatchArrivalIdx].cost & 0x3fffffff; + int nCodingChoiceCost = nPrevCost /* the actual cost of the literals themselves accumulates up the chain */ + nMatchLenCost + nNoRepmatchOffsetCost; + + if (!cur_arrival[nNonRepMatchArrivalIdx].num_literals) + nCodingChoiceCost += nModeSwitchPenalty; + + int nScore = cur_arrival[nNonRepMatchArrivalIdx].score + nScorePenalty; + if (nCodingChoiceCost < pDestSlots[nArrivalsPerPosition - 2].cost || + (nCodingChoiceCost == pDestSlots[nArrivalsPerPosition - 2].cost && nScore < (pDestSlots[nArrivalsPerPosition - 2].score + nDisableScore))) { + int exists = 0; + + for (n = 0; + n < nArrivalsPerPosition && pDestSlots[n].cost < nCodingChoiceCost; + n++) { + if (pDestSlots[n].rep_offset == nMatchOffset) { + exists = 1; + break; + } + } + + if (!exists) { + for (; + n < nArrivalsPerPosition && pDestSlots[n].cost == nCodingChoiceCost && nScore >= (pDestSlots[n].score + nDisableScore); + n++) { + if (pDestSlots[n].rep_offset == nMatchOffset) { + exists = 1; + break; + } + } + + if (!exists) { + if (n < nArrivalsPerPosition - 1) { + int nn; + + for (nn = n; + nn < nArrivalsPerPosition && pDestSlots[nn].cost == nCodingChoiceCost; + nn++) { + if (pDestSlots[nn].rep_offset == nMatchOffset && + (!nInsertForwardReps || pDestSlots[nn].rep_pos >= i || + pDestSlots[nArrivalsPerPosition - 1].from_slot)) { + exists = 1; + break; + } + } + + if (!exists) { + int z; + + for (z = n; z < nArrivalsPerPosition - 1 && pDestSlots[z].from_slot; z++) { + if (pDestSlots[z].rep_offset == nMatchOffset) + break; + } + + if (z == (nArrivalsPerPosition - 1) && pDestSlots[z].from_slot && pDestSlots[z].match_len < MIN_MATCH_SIZE_V2) + z--; + + memmove(&pDestSlots[n + 1], + &pDestSlots[n], + sizeof(lzsa_arrival) * (z - n)); + + lzsa_arrival* pDestArrival = &pDestSlots[n]; + pDestArrival->cost = nCodingChoiceCost; + pDestArrival->from_pos = i; + pDestArrival->from_slot = nNonRepMatchArrivalIdx + 1; + pDestArrival->match_len = k; + pDestArrival->num_literals = 0; + pDestArrival->score = nScore; + pDestArrival->rep_offset = nMatchOffset; + pDestArrival->rep_pos = i; + pDestArrival->rep_len = k; + } + } + } + } + } + } + + /* Insert repmatch candidates */ + + if (k > nMinOverallRepLen && k <= nMaxOverallRepLen) { + for (j = 0; j < nNumArrivalsForThisPos; j++) { + int nMaskOffset = (j << 7) + (k >> 3); + if (nRepLenForArrival[j] >= k && (nReduce || !(nRepLenHandledMask[nMaskOffset] & (1 << (k & 7))))) { + const int nPrevCost = cur_arrival[j].cost & 0x3fffffff; + int nRepCodingChoiceCost = nPrevCost /* the actual cost of the literals themselves accumulates up the chain */ + nMatchLenCost; + int nScore = cur_arrival[j].score + 2; + + if (nRepCodingChoiceCost < pDestSlots[nArrivalsPerPosition - 1].cost || + (nRepCodingChoiceCost == pDestSlots[nArrivalsPerPosition - 1].cost && nScore < (pDestSlots[nArrivalsPerPosition - 1].score + nDisableScore))) { + int nRepOffset = cur_arrival[j].rep_offset; + int exists = 0; + + for (n = 0; + n < nArrivalsPerPosition && pDestSlots[n].cost < nRepCodingChoiceCost; + n++) { + if (pDestSlots[n].rep_offset == nRepOffset) { + exists = 1; + if (!nReduce) + nRepLenHandledMask[nMaskOffset] |= 1 << (k & 7); + break; + } + } + + if (!exists) { + for (; + n < nArrivalsPerPosition && pDestSlots[n].cost == nRepCodingChoiceCost && nScore >= (pDestSlots[n].score + nDisableScore); + n++) { + if (pDestSlots[n].rep_offset == nRepOffset) { + exists = 1; + break; + } + } + + if (!exists) { + if (n < nArrivalsPerPosition) { + int nn; + + for (nn = n; + nn < nArrivalsPerPosition && pDestSlots[nn].cost == nRepCodingChoiceCost; + nn++) { + if (pDestSlots[nn].rep_offset == nRepOffset) { + exists = 1; + break; + } + } + + if (!exists) { + int z; + + for (z = n; z < nArrivalsPerPosition - 1 && pDestSlots[z].from_slot; z++) { + if (pDestSlots[z].rep_offset == nRepOffset) + break; + } + + memmove(&pDestSlots[n + 1], + &pDestSlots[n], + sizeof(lzsa_arrival) * (z - n)); + + lzsa_arrival* pDestArrival = &pDestSlots[n]; + pDestArrival->cost = nRepCodingChoiceCost; + pDestArrival->from_pos = i; + pDestArrival->from_slot = j + 1; + pDestArrival->match_len = k; + pDestArrival->num_literals = 0; + pDestArrival->score = nScore; + pDestArrival->rep_offset = nRepOffset; + pDestArrival->rep_pos = i; + pDestArrival->rep_len = k; + } + } + } + } + } + else { + break; + } + } + } + + if (k < nMaxRepInsertedLen) + nMinOverallRepLen = k; + } + } + + if (nMatchLen >= LCP_MAX && ((m + 1) >= NMATCHES_PER_INDEX_V2 || match[m + 1].length < LCP_MAX)) + break; + } + } + + lzsa_arrival *end_arrival = &arrival[(i << ARRIVALS_PER_POSITION_SHIFT) + 0]; + + while (end_arrival->from_slot > 0 && end_arrival->from_pos >= 0) { + if (end_arrival->from_pos >= nEndOffset) return; + pBestMatch[end_arrival->from_pos].length = end_arrival->match_len; + if (end_arrival->match_len) + pBestMatch[end_arrival->from_pos].offset = end_arrival->rep_offset; + else + pBestMatch[end_arrival->from_pos].offset = 0; + end_arrival = &arrival[(end_arrival->from_pos << ARRIVALS_PER_POSITION_SHIFT) + (end_arrival->from_slot - 1)]; + } +} + +/** + * Attempt to minimize the number of commands issued in the compressed data block, in order to speed up decompression without + * impacting the compression ratio + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param pBestMatch optimal matches to evaluate and update + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * + * @return non-zero if the number of tokens was reduced, 0 if it wasn't + */ +static int lzsa_optimize_command_count_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) { + int i; + int nNumLiterals = 0; + int nPrevRepMatchOffset = 0; + int nRepMatchOffset = 0; + int nRepMatchLen = 0; + int nRepIndex = 0; + int nDidReduce = 0; + + for (i = nStartOffset; i < nEndOffset; ) { + lzsa_match *pMatch = pBestMatch + i; + + if (pMatch->length == 0 && + (i + 1) < nEndOffset && + pBestMatch[i + 1].length >= MIN_MATCH_SIZE_V2 && + pBestMatch[i + 1].length < MAX_VARLEN && + pBestMatch[i + 1].offset && + i >= pBestMatch[i + 1].offset && + (i + pBestMatch[i + 1].length + 1) <= nEndOffset && + !memcmp(pInWindow + i - (pBestMatch[i + 1].offset), pInWindow + i, pBestMatch[i + 1].length + 1)) { + int nCurLenSize = lzsa_get_match_varlen_size_v2(pBestMatch[i + 1].length - MIN_MATCH_SIZE_V2); + int nReducedLenSize = lzsa_get_match_varlen_size_v2(pBestMatch[i + 1].length + 1 - MIN_MATCH_SIZE_V2); + + if ((nReducedLenSize - nCurLenSize) <= 8) { + /* Merge */ + pBestMatch[i].length = pBestMatch[i + 1].length + 1; + pBestMatch[i].offset = pBestMatch[i + 1].offset; + pBestMatch[i + 1].length = 0; + pBestMatch[i + 1].offset = 0; + nDidReduce = 1; + continue; + } + } + + if (pMatch->length >= MIN_MATCH_SIZE_V2) { + if ((i + pMatch->length) < nEndOffset /* Don't consider the last match in the block, we can only reduce a match inbetween other tokens */) { + int nNextIndex = i + pMatch->length; + int nNextLiterals = 0; + + while (nNextIndex < nEndOffset && pBestMatch[nNextIndex].length < MIN_MATCH_SIZE_V2) { + nNextLiterals++; + nNextIndex++; + } + + if (nNextIndex < nEndOffset && pBestMatch[nNextIndex].length >= MIN_MATCH_SIZE_V2) { + /* This command is a match, is followed by 'nNextLiterals' literals and then by another match */ + + if (nRepMatchOffset && pMatch->offset != nRepMatchOffset && (pBestMatch[nNextIndex].offset != pMatch->offset || pBestMatch[nNextIndex].offset == nRepMatchOffset || + ((pMatch->offset <= 32) ? 4 : ((pMatch->offset <= 512) ? 8 : ((pMatch->offset <= (8192 + 512)) ? 12 : 16))) > + ((pBestMatch[nNextIndex].offset <= 32) ? 4 : ((pBestMatch[nNextIndex].offset <= 512) ? 8 : ((pBestMatch[nNextIndex].offset <= (8192 + 512)) ? 12 : 16))))) { + /* Check if we can change the current match's offset to be the same as the previous match's offset, and get an extra repmatch. This will occur when + * matching large regions of identical bytes for instance, where there are too many offsets to be considered by the parser, and when not compressing to favor the + * ratio (the forward arrivals parser already has this covered). */ + if (i > nRepMatchOffset && + (i - nRepMatchOffset + pMatch->length) <= nEndOffset && + !memcmp(pInWindow + i - nRepMatchOffset, pInWindow + i - pMatch->offset, pMatch->length)) { + pMatch->offset = nRepMatchOffset; + nDidReduce = 1; + } + } + + if (pBestMatch[nNextIndex].offset && pMatch->offset != pBestMatch[nNextIndex].offset && nRepMatchOffset != pBestMatch[nNextIndex].offset) { + /* Otherwise, try to gain a match forward as well */ + if (i > pBestMatch[nNextIndex].offset && (i - pBestMatch[nNextIndex].offset + pMatch->length) <= nEndOffset) { + int nMaxLen = 0; + while (nMaxLen < pMatch->length && pInWindow[i - pBestMatch[nNextIndex].offset + nMaxLen] == pInWindow[i - pMatch->offset + nMaxLen]) + nMaxLen++; + if (nMaxLen >= pMatch->length) { + /* Replace */ + pMatch->offset = pBestMatch[nNextIndex].offset; + nDidReduce = 1; + } + else if (nMaxLen >= 2 && pMatch->offset != nRepMatchOffset) { + int nPartialSizeBefore, nPartialSizeAfter; + + nPartialSizeBefore = lzsa_get_match_varlen_size_v2(pMatch->length - MIN_MATCH_SIZE_V2); + nPartialSizeBefore += (pMatch->offset <= 32) ? 4 : ((pMatch->offset <= 512) ? 8 : ((pMatch->offset <= (8192 + 512)) ? 12 : 16)); + nPartialSizeBefore += lzsa_get_literals_varlen_size_v2(nNextLiterals); + + nPartialSizeAfter = lzsa_get_match_varlen_size_v2(nMaxLen - MIN_MATCH_SIZE_V2); + nPartialSizeAfter += lzsa_get_literals_varlen_size_v2(nNextLiterals + (pMatch->length - nMaxLen)) + ((pMatch->length - nMaxLen) << 3); + + if (nPartialSizeAfter < nPartialSizeBefore) { + int j; + + /* We gain a repmatch that is shorter than the original match as this is the best we can do, so it is followed by extra literals, but + * we have calculated that this is shorter */ + pMatch->offset = pBestMatch[nNextIndex].offset; + for (j = nMaxLen; j < pMatch->length; j++) { + pBestMatch[i + j].length = 0; + } + pMatch->length = nMaxLen; + nDidReduce = 1; + } + } + } + } + + if (pMatch->length < 9 /* Don't waste time considering large matches, they will always win over literals */) { + /* Calculate this command's current cost (excluding 'nNumLiterals' bytes) */ + + int nCurCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + lzsa_get_match_varlen_size_v2(pMatch->length - MIN_MATCH_SIZE_V2); + if (pMatch->offset != nRepMatchOffset) + nCurCommandSize += (pMatch->offset <= 32) ? 4 : ((pMatch->offset <= 512) ? 8 : ((pMatch->offset <= (8192 + 512)) ? 12 : 16)); + + /* Calculate the next command's current cost */ + int nNextCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNextLiterals) + /* (nNextLiterals << 3) + */ lzsa_get_match_varlen_size_v2(pBestMatch[nNextIndex].length - MIN_MATCH_SIZE_V2); + if (pBestMatch[nNextIndex].offset != pMatch->offset) + nNextCommandSize += (pBestMatch[nNextIndex].offset <= 32) ? 4 : ((pBestMatch[nNextIndex].offset <= 512) ? 8 : ((pBestMatch[nNextIndex].offset <= (8192 + 512)) ? 12 : 16)); + + int nOriginalCombinedCommandSize = nCurCommandSize + nNextCommandSize; + + /* Calculate the cost of replacing this match command by literals + the next command with the cost of encoding these literals (excluding 'nNumLiterals' bytes) */ + int nReducedCommandSize = (pMatch->length << 3) + 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals + pMatch->length + nNextLiterals) + /* (nNextLiterals << 3) + */ lzsa_get_match_varlen_size_v2(pBestMatch[nNextIndex].length - MIN_MATCH_SIZE_V2); + if (pBestMatch[nNextIndex].offset != nRepMatchOffset) + nReducedCommandSize += (pBestMatch[nNextIndex].offset <= 32) ? 4 : ((pBestMatch[nNextIndex].offset <= 512) ? 8 : ((pBestMatch[nNextIndex].offset <= (8192 + 512)) ? 12 : 16)); + + int nReplaceRepOffset = 0; + if (nRepMatchOffset && nRepMatchOffset != nPrevRepMatchOffset && nRepMatchLen >= MIN_MATCH_SIZE_V2 && nRepMatchOffset != pBestMatch[nNextIndex].offset && nRepIndex > pBestMatch[nNextIndex].offset && + (nRepIndex - pBestMatch[nNextIndex].offset + nRepMatchLen) <= nEndOffset && + !memcmp(pInWindow + nRepIndex - nRepMatchOffset, pInWindow + nRepIndex - pBestMatch[nNextIndex].offset, nRepMatchLen)) { + /* Replacing this match command by literals would let us create a repmatch */ + nReplaceRepOffset = 1; + nReducedCommandSize -= (nRepMatchOffset <= 32) ? 4 : ((nRepMatchOffset <= 512) ? 8 : ((nRepMatchOffset <= (8192 + 512)) ? 12 : 16)); + } + + if (nOriginalCombinedCommandSize >= nReducedCommandSize) { + /* Reduce */ + int nMatchLen = pMatch->length; + int j; + + for (j = 0; j < nMatchLen; j++) { + pBestMatch[i + j].length = 0; + } + + nDidReduce = 1; + + if (nReplaceRepOffset) { + pBestMatch[nRepIndex].offset = pBestMatch[nNextIndex].offset; + nRepMatchOffset = pBestMatch[nNextIndex].offset; + } + continue; + } + } + } + } + + if ((i + pMatch->length) <= nEndOffset && pMatch->offset > 0 && pMatch->length >= MIN_MATCH_SIZE_V2 && + pBestMatch[i + pMatch->length].offset > 0 && + pBestMatch[i + pMatch->length].length >= MIN_MATCH_SIZE_V2 && + (pMatch->length + pBestMatch[i + pMatch->length].length) >= LEAVE_ALONE_MATCH_SIZE && + (pMatch->length + pBestMatch[i + pMatch->length].length) <= MAX_VARLEN && + (i + pMatch->length) > pMatch->offset && + (i + pMatch->length) > pBestMatch[i + pMatch->length].offset && + (i + pMatch->length + pBestMatch[i + pMatch->length].length) <= nEndOffset && + !memcmp(pInWindow + i - pMatch->offset + pMatch->length, + pInWindow + i + pMatch->length - pBestMatch[i + pMatch->length].offset, + pBestMatch[i + pMatch->length].length)) { + + int nNextIndex = i + pMatch->length; + + while (nNextIndex < nEndOffset && pBestMatch[nNextIndex].length < MIN_MATCH_SIZE_V2) { + nNextIndex++; + } + + int nNextOffset; + if (nNextIndex < nEndOffset) + nNextOffset = pBestMatch[nNextIndex].offset; + else + nNextOffset = 0; + + int nCurPartialSize = lzsa_get_match_varlen_size_v2(pMatch->length - MIN_MATCH_SIZE_V2); + + nCurPartialSize += 8 /* token */ + /* lzsa_get_literals_varlen_size_v2(0) + */ lzsa_get_match_varlen_size_v2(pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V2); + if (pBestMatch[i + pMatch->length].offset != pMatch->offset) + nCurPartialSize += (pBestMatch[i + pMatch->length].offset <= 32) ? 4 : ((pBestMatch[i + pMatch->length].offset <= 512) ? 8 : ((pBestMatch[i + pMatch->length].offset <= (8192 + 512)) ? 12 : 16)); + + if (nNextOffset != pBestMatch[i + pMatch->length].offset) + nCurPartialSize += (nNextOffset <= 32) ? 4 : ((nNextOffset <= 512) ? 8 : ((nNextOffset <= (8192 + 512)) ? 12 : 16)); + + int nReducedPartialSize = lzsa_get_match_varlen_size_v2(pMatch->length + pBestMatch[i + pMatch->length].length - MIN_MATCH_SIZE_V2); + + if (nNextOffset != pMatch->offset) + nReducedPartialSize += (nNextOffset <= 32) ? 4 : ((nNextOffset <= 512) ? 8 : ((nNextOffset <= (8192 + 512)) ? 12 : 16)); + + if (nCurPartialSize >= nReducedPartialSize) { + int nMatchLen = pMatch->length; + + /* Join */ + + pMatch->length += pBestMatch[i + nMatchLen].length; + pBestMatch[i + nMatchLen].offset = 0; + pBestMatch[i + nMatchLen].length = -1; + nDidReduce = 1; + continue; + } + } + + nPrevRepMatchOffset = nRepMatchOffset; + nRepMatchOffset = pMatch->offset; + nRepMatchLen = pMatch->length; + nRepIndex = i; + + i += pMatch->length; + nNumLiterals = 0; + } + else { + nNumLiterals++; + i++; + } + } + + return nDidReduce; +} + +/** + * Get compressed data block size + * + * @param pCompressor compression context + * @param pBestMatch optimal matches to emit + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * + * @return size of compressed data that will be written to output buffer + */ +static int lzsa_get_compressed_size_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const int nStartOffset, const int nEndOffset) { + int i; + int nNumLiterals = 0; + int nRepMatchOffset = 0; + int nCompressedSize = 0; + + for (i = nStartOffset; i < nEndOffset; ) { + const lzsa_match *pMatch = pBestMatch + i; + + if (pMatch->length >= MIN_MATCH_SIZE_V2) { + int nMatchOffset = pMatch->offset; + int nMatchLen = pMatch->length; + int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V2; + int nOffsetSize; + + if (nMatchOffset == nRepMatchOffset) { + nOffsetSize = 0; + } + else { + if (nMatchOffset <= 32) { + nOffsetSize = 4; + } + else if (nMatchOffset <= 512) { + nOffsetSize = 8; + } + else if (nMatchOffset <= (8192 + 512)) { + nOffsetSize = 12; + } + else { + nOffsetSize = 16; + } + } + + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + nOffsetSize /* match offset */ + lzsa_get_match_varlen_size_v2(nEncodedMatchLen); + nCompressedSize += nCommandSize; + + nNumLiterals = 0; + nRepMatchOffset = nMatchOffset; + i += nMatchLen; + } + else { + nNumLiterals++; + i++; + } + } + + { + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3); + + nCompressedSize += nCommandSize; + nNumLiterals = 0; + } + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + nCompressedSize += (8 + 4); + } + + return nCompressedSize; +} + +/** + * Emit block of compressed data + * + * @param pCompressor compression context + * @param pBestMatch optimal matches to emit + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +static int lzsa_write_block_v2(lzsa_compressor *pCompressor, lzsa_match *pBestMatch, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) { + int i; + int nNumLiterals = 0; + int nInFirstLiteralOffset = 0; + int nOutOffset = 0; + int nCurNibbleOffset = -1; + int nRepMatchOffset = 0; + + for (i = nStartOffset; i < nEndOffset; ) { + const lzsa_match *pMatch = pBestMatch + i; + + if (pMatch->length >= MIN_MATCH_SIZE_V2) { + int nMatchOffset = pMatch->offset; + int nMatchLen = pMatch->length; + int nEncodedMatchLen = nMatchLen - MIN_MATCH_SIZE_V2; + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals; + int nTokenMatchLen = (nEncodedMatchLen >= MATCH_RUN_LEN_V2) ? MATCH_RUN_LEN_V2 : nEncodedMatchLen; + int nTokenOffsetMode; + int nOffsetSize; + + if (nMatchOffset == nRepMatchOffset) { + nTokenOffsetMode = 0xe0; + nOffsetSize = 0; + } + else { + if (nMatchOffset <= 32) { + nTokenOffsetMode = 0x00 | ((((-nMatchOffset) & 0x01) << 5) ^ 0x20); + nOffsetSize = 4; + } + else if (nMatchOffset <= 512) { + nTokenOffsetMode = 0x40 | ((((-nMatchOffset) & 0x100) >> 3) ^ 0x20); + nOffsetSize = 8; + } + else if (nMatchOffset <= (8192 + 512)) { + nTokenOffsetMode = 0x80 | ((((-(nMatchOffset - 512)) & 0x0100) >> 3) ^ 0x20); + nOffsetSize = 12; + } + else { + nTokenOffsetMode = 0xc0; + nOffsetSize = 16; + } + } + + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + nOffsetSize /* match offset */ + lzsa_get_match_varlen_size_v2(nEncodedMatchLen); + + if ((nOutOffset + ((nCommandSize + 7) >> 3)) > nMaxOutDataSize) + return -1; + if (nMatchOffset < MIN_OFFSET || nMatchOffset > MAX_OFFSET) + return -1; + + pOutData[nOutOffset++] = nTokenOffsetMode | (nTokenLiteralsLen << 3) | nTokenMatchLen; + nOutOffset = lzsa_write_literals_varlen_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, nNumLiterals); + if (nOutOffset < 0) return -1; + + if (nNumLiterals < pCompressor->stats.min_literals || pCompressor->stats.min_literals == -1) + pCompressor->stats.min_literals = nNumLiterals; + if (nNumLiterals > pCompressor->stats.max_literals) + pCompressor->stats.max_literals = nNumLiterals; + pCompressor->stats.total_literals += nNumLiterals; + pCompressor->stats.literals_divisor++; + + if (nNumLiterals != 0) { + memcpy(pOutData + nOutOffset, pInWindow + nInFirstLiteralOffset, nNumLiterals); + nOutOffset += nNumLiterals; + nNumLiterals = 0; + } + + if (nTokenOffsetMode == 0x00 || nTokenOffsetMode == 0x20) { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, ((-nMatchOffset) & 0x1e) >> 1); + if (nOutOffset < 0) return -1; + } + else if (nTokenOffsetMode == 0x40 || nTokenOffsetMode == 0x60) { + pOutData[nOutOffset++] = (-nMatchOffset) & 0xff; + } + else if (nTokenOffsetMode == 0x80 || nTokenOffsetMode == 0xa0) { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, ((-(nMatchOffset - 512)) >> 9) & 0x0f); + if (nOutOffset < 0) return -1; + pOutData[nOutOffset++] = (-(nMatchOffset - 512)) & 0xff; + } + else if (nTokenOffsetMode == 0xc0) { + pOutData[nOutOffset++] = (-nMatchOffset) >> 8; + pOutData[nOutOffset++] = (-nMatchOffset) & 0xff; + } + + if (nMatchOffset == nRepMatchOffset) + pCompressor->stats.num_rep_offsets++; + + nRepMatchOffset = nMatchOffset; + + nOutOffset = lzsa_write_match_varlen_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, nEncodedMatchLen); + if (nOutOffset < 0) return -1; + + if (nMatchOffset < pCompressor->stats.min_offset || pCompressor->stats.min_offset == -1) + pCompressor->stats.min_offset = nMatchOffset; + if (nMatchOffset > pCompressor->stats.max_offset) + pCompressor->stats.max_offset = nMatchOffset; + pCompressor->stats.total_offsets += nMatchOffset; + + if (nMatchLen < pCompressor->stats.min_match_len || pCompressor->stats.min_match_len == -1) + pCompressor->stats.min_match_len = nMatchLen; + if (nMatchLen > pCompressor->stats.max_match_len) + pCompressor->stats.max_match_len = nMatchLen; + pCompressor->stats.total_match_lens += nMatchLen; + pCompressor->stats.match_divisor++; + + if (nMatchOffset == 1) { + if (nMatchLen < pCompressor->stats.min_rle1_len || pCompressor->stats.min_rle1_len == -1) + pCompressor->stats.min_rle1_len = nMatchLen; + if (nMatchLen > pCompressor->stats.max_rle1_len) + pCompressor->stats.max_rle1_len = nMatchLen; + pCompressor->stats.total_rle1_lens += nMatchLen; + pCompressor->stats.rle1_divisor++; + } + else if (nMatchOffset == 2) { + if (nMatchLen < pCompressor->stats.min_rle2_len || pCompressor->stats.min_rle2_len == -1) + pCompressor->stats.min_rle2_len = nMatchLen; + if (nMatchLen > pCompressor->stats.max_rle2_len) + pCompressor->stats.max_rle2_len = nMatchLen; + pCompressor->stats.total_rle2_lens += nMatchLen; + pCompressor->stats.rle2_divisor++; + } + + i += nMatchLen; + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + int nCurSafeDist = (i - nStartOffset) - nOutOffset; + if (nCurSafeDist >= 0 && pCompressor->safe_dist < nCurSafeDist) + pCompressor->safe_dist = nCurSafeDist; + } + + pCompressor->num_commands++; + } + else { + if (nNumLiterals == 0) + nInFirstLiteralOffset = i; + nNumLiterals++; + i++; + } + } + + { + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals; + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3); + + if ((nOutOffset + ((nCommandSize + 7) >> 3)) > nMaxOutDataSize) + return -1; + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) + pOutData[nOutOffset++] = (nTokenLiteralsLen << 3) | 0xe7; + else + pOutData[nOutOffset++] = (nTokenLiteralsLen << 3) | 0x00; + nOutOffset = lzsa_write_literals_varlen_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, nNumLiterals); + if (nOutOffset < 0) return -1; + + if (nNumLiterals < pCompressor->stats.min_literals || pCompressor->stats.min_literals == -1) + pCompressor->stats.min_literals = nNumLiterals; + if (nNumLiterals > pCompressor->stats.max_literals) + pCompressor->stats.max_literals = nNumLiterals; + pCompressor->stats.total_literals += nNumLiterals; + pCompressor->stats.literals_divisor++; + + if (nNumLiterals != 0) { + memcpy(pOutData + nOutOffset, pInWindow + nInFirstLiteralOffset, nNumLiterals); + nOutOffset += nNumLiterals; + nNumLiterals = 0; + } + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + int nCurSafeDist = (i - nStartOffset) - nOutOffset; + if (nCurSafeDist >= 0 && pCompressor->safe_dist < nCurSafeDist) + pCompressor->safe_dist = nCurSafeDist; + } + + pCompressor->num_commands++; + } + + if (pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + /* Emit EOD marker for raw block */ + + if (nOutOffset >= nMaxOutDataSize) + return -1; + + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, 15); /* Extended match length nibble */ + if (nOutOffset < 0) return -1; + + if ((nOutOffset + 1) > nMaxOutDataSize) + return -1; + + pOutData[nOutOffset++] = 232; /* EOD match length byte */ + } + + if (nCurNibbleOffset != -1) { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, 0); + if (nOutOffset < 0 || nCurNibbleOffset != -1) + return -1; + } + + return nOutOffset; +} + +/** + * Emit raw block of uncompressible data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nStartOffset current offset in input window (typically the number of previously compressed bytes) + * @param nEndOffset offset to end finding matches at (typically the size of the total input window in bytes + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +static int lzsa_write_raw_uncompressed_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nStartOffset, const int nEndOffset, unsigned char *pOutData, const int nMaxOutDataSize) { + int nCurNibbleOffset = -1; + int nNumLiterals = nEndOffset - nStartOffset; + int nTokenLiteralsLen = (nNumLiterals >= LITERALS_RUN_LEN_V2) ? LITERALS_RUN_LEN_V2 : nNumLiterals; + int nOutOffset = 0; + + int nCommandSize = 8 /* token */ + lzsa_get_literals_varlen_size_v2(nNumLiterals) + (nNumLiterals << 3) + 4 + 8; + if ((nOutOffset + ((nCommandSize + 7) >> 3)) > nMaxOutDataSize) + return -1; + + pCompressor->num_commands = 0; + pOutData[nOutOffset++] = (nTokenLiteralsLen << 3) | 0xe7; + + nOutOffset = lzsa_write_literals_varlen_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, nNumLiterals); + if (nOutOffset < 0) return -1; + + if (nNumLiterals != 0) { + memcpy(pOutData + nOutOffset, pInWindow + nStartOffset, nNumLiterals); + nOutOffset += nNumLiterals; + nNumLiterals = 0; + } + + /* Emit EOD marker for raw block */ + + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, 15); /* Extended match length nibble */ + if (nOutOffset < 0) return -1; + + if ((nOutOffset + 1) > nMaxOutDataSize) + return -1; + + pOutData[nOutOffset++] = 232; /* EOD match length byte */ + + pCompressor->num_commands++; + + if (nCurNibbleOffset != -1) { + nOutOffset = lzsa_write_nibble_v2(pOutData, nOutOffset, nMaxOutDataSize, &nCurNibbleOffset, 0); + if (nOutOffset < 0 || nCurNibbleOffset != -1) + return -1; + } + + return nOutOffset; +} + +/** + * Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA2 data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nPreviousBlockSize number of previously compressed bytes (or 0 for none) + * @param nInDataSize number of input bytes to compress + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { + int nResult, nBaseCompressedSize; + int nArrivalsPerPosition = (nInDataSize < 65536) ? NARRIVALS_PER_POSITION_V2_BIG : NARRIVALS_PER_POSITION_V2_SMALL; + int *rle_end = (int*)pCompressor->intervals /* reuse */; + int i; + + i = 0; + while (i < (nPreviousBlockSize + nInDataSize)) { + int nRangeStartIdx = i; + unsigned char c = pInWindow[nRangeStartIdx]; + do { + i++; + } while (i < (nPreviousBlockSize + nInDataSize) && pInWindow[i] == c); + while (nRangeStartIdx < i) { + rle_end[nRangeStartIdx++] = i; + } + } + + /* Compress optimally without breaking ties in favor of less tokens */ + + memset(pCompressor->best_match, 0, BLOCK_SIZE * sizeof(lzsa_match)); + lzsa_optimize_forward_v2(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 0 /* reduce */, (nInDataSize < 65536) ? 1 : 0 /* insert forward reps */, nArrivalsPerPosition); + + int nDidReduce; + int nPasses = 0; + do { + nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + nPasses++; + } while (nDidReduce && nPasses < 20); + + nBaseCompressedSize = lzsa_get_compressed_size_v2(pCompressor, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + lzsa_match *pBestMatch = pCompressor->best_match - nPreviousBlockSize; + + if (nBaseCompressedSize > 0 && nInDataSize < 65536) { + int nReducedCompressedSize; + + /* Compress optimally and do break ties in favor of less tokens */ + memset(pCompressor->improved_match, 0, BLOCK_SIZE * sizeof(lzsa_match)); + lzsa_optimize_forward_v2(pCompressor, pInWindow, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 1 /* reduce */, 0 /* use forward reps */, nArrivalsPerPosition); + + nPasses = 0; + do { + nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pInWindow, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + nPasses++; + } while (nDidReduce && nPasses < 20); + + nReducedCompressedSize = lzsa_get_compressed_size_v2(pCompressor, pCompressor->improved_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + if (nReducedCompressedSize > 0 && nReducedCompressedSize <= nBaseCompressedSize) { + const int nEndOffset = nPreviousBlockSize + nInDataSize; + int nSupplementedCompressedSize; + + /* Pick the parse with the reduced number of tokens as it didn't negatively affect the size */ + pBestMatch = pCompressor->improved_match - nPreviousBlockSize; + + int* first_offset_for_byte = pCompressor->first_offset_for_byte; + int* next_offset_for_pos = pCompressor->next_offset_for_pos; + int nPosition; + + /* Supplement small matches */ + + memset(first_offset_for_byte, 0xff, sizeof(int) * 65536); + memset(next_offset_for_pos, 0xff, sizeof(int) * nInDataSize); + + for (nPosition = nPreviousBlockSize; nPosition < nEndOffset - 1; nPosition++) { + next_offset_for_pos[nPosition - nPreviousBlockSize] = first_offset_for_byte[((unsigned int)pInWindow[nPosition]) | (((unsigned int)pInWindow[nPosition + 1]) << 8)]; + first_offset_for_byte[((unsigned int)pInWindow[nPosition]) | (((unsigned int)pInWindow[nPosition + 1]) << 8)] = nPosition; + } + + for (nPosition = nPreviousBlockSize + 1; nPosition < (nEndOffset - 1); nPosition++) { + lzsa_match* match = pCompressor->match + ((nPosition - nPreviousBlockSize) << MATCHES_PER_INDEX_SHIFT_V2); + int m = 0, nInserted = 0; + int nMatchPos; + + while (m < 15 && match[m].length) + m++; + + for (nMatchPos = next_offset_for_pos[nPosition - nPreviousBlockSize]; m < 15 && nMatchPos >= 0; nMatchPos = next_offset_for_pos[nMatchPos - nPreviousBlockSize]) { + int nMatchOffset = nPosition - nMatchPos; + int nExistingMatchIdx; + int nAlreadyExists = 0; + + for (nExistingMatchIdx = 0; nExistingMatchIdx < m; nExistingMatchIdx++) { + if (match[nExistingMatchIdx].offset == nMatchOffset) { + nAlreadyExists = 1; + break; + } + } + + if (!nAlreadyExists) { + int nMatchLen = 2; + while (nMatchLen < 16 && (nPosition + nMatchLen + 4) < nEndOffset && !memcmp(pInWindow + nMatchPos + nMatchLen, pInWindow + nPosition + nMatchLen, 4)) + nMatchLen += 4; + while (nMatchLen < 16 && (nPosition + nMatchLen) < nEndOffset && pInWindow[nMatchPos + nMatchLen] == pInWindow[nPosition + nMatchLen]) + nMatchLen++; + match[m].length = nMatchLen; + match[m].offset = nMatchOffset; + m++; + nInserted++; + if (nInserted >= 15) + break; + } + } + } + + /* Compress optimally with the extra matches */ + memset(pCompressor->best_match, 0, BLOCK_SIZE * sizeof(lzsa_match)); + lzsa_optimize_forward_v2(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, 1 /* reduce */, 0 /* use forward reps */, nArrivalsPerPosition); + + nPasses = 0; + do { + nDidReduce = lzsa_optimize_command_count_v2(pCompressor, pInWindow, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + nPasses++; + } while (nDidReduce && nPasses < 20); + + nSupplementedCompressedSize = lzsa_get_compressed_size_v2(pCompressor, pCompressor->best_match - nPreviousBlockSize, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + if (nSupplementedCompressedSize > 0 && nSupplementedCompressedSize < nReducedCompressedSize) { + /* Pick the parse with the extra matches as it didn't negatively affect the size */ + pBestMatch = pCompressor->best_match - nPreviousBlockSize; + } + } + } + + nResult = lzsa_write_block_v2(pCompressor, pBestMatch, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + if (nResult < 0 && pCompressor->flags & LZSA_FLAG_RAW_BLOCK) { + nResult = lzsa_write_raw_uncompressed_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nPreviousBlockSize + nInDataSize, pOutData, nMaxOutDataSize); + } + + return nResult; +} diff --git a/tools/rasm/lzsa-master/src/shrink_block_v2.h b/tools/rasm/lzsa-master/src/shrink_block_v2.h new file mode 100644 index 0000000..4a83608 --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_block_v2.h @@ -0,0 +1,53 @@ +/* + * shrink_block_v2.h - LZSA2 block compressor definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _SHRINK_BLOCK_V2_H +#define _SHRINK_BLOCK_V2_H + +/* Forward declarations */ +typedef struct _lzsa_compressor lzsa_compressor; + +/** + * Select the most optimal matches, reduce the token count if possible, and then emit a block of compressed LZSA2 data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nPreviousBlockSize number of previously compressed bytes (or 0 for none) + * @param nInDataSize number of input bytes to compress + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +int lzsa_optimize_and_write_block_v2(lzsa_compressor *pCompressor, const unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize); + +#endif /* _SHRINK_BLOCK_V2_H */ diff --git a/tools/rasm/lzsa-master/src/shrink_context.c b/tools/rasm/lzsa-master/src/shrink_context.c new file mode 100644 index 0000000..0bb8a6d --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_context.c @@ -0,0 +1,254 @@ +/* + * shrink_context.c - compression context implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "shrink_context.h" +#include "shrink_block_v1.h" +#include "shrink_block_v2.h" +#include "format.h" +#include "matchfinder.h" +#include "lib.h" + +/** + * Initialize compression context + * + * @param pCompressor compression context to initialize + * @param nMaxWindowSize maximum size of input data window (previously compressed bytes + bytes to compress) + * @param nMinMatchSize minimum match size (cannot be less than MIN_MATCH_SIZE) + * @param nFlags compression flags + * + * @return 0 for success, non-zero for failure + */ +int lzsa_compressor_init(lzsa_compressor *pCompressor, const int nMaxWindowSize, const int nMinMatchSize, const int nFormatVersion, const int nFlags) { + int nResult; + int nMinMatchSizeForFormat = (nFormatVersion == 1) ? MIN_MATCH_SIZE_V1 : MIN_MATCH_SIZE_V2; + int nMaxMinMatchForFormat = (nFormatVersion == 1) ? 5 : 3; + + nResult = divsufsort_init(&pCompressor->divsufsort_context); + pCompressor->intervals = NULL; + pCompressor->pos_data = NULL; + pCompressor->open_intervals = NULL; + pCompressor->match = NULL; + pCompressor->best_match = NULL; + pCompressor->improved_match = NULL; + pCompressor->arrival = NULL; + pCompressor->rep_handled_mask = NULL; + pCompressor->first_offset_for_byte = NULL; + pCompressor->next_offset_for_pos = NULL; + pCompressor->min_match_size = nMinMatchSize; + if (pCompressor->min_match_size < nMinMatchSizeForFormat) + pCompressor->min_match_size = nMinMatchSizeForFormat; + else if (pCompressor->min_match_size > nMaxMinMatchForFormat) + pCompressor->min_match_size = nMaxMinMatchForFormat; + pCompressor->format_version = nFormatVersion; + pCompressor->flags = nFlags; + pCompressor->safe_dist = 0; + pCompressor->num_commands = 0; + + memset(&pCompressor->stats, 0, sizeof(pCompressor->stats)); + pCompressor->stats.min_literals = -1; + pCompressor->stats.min_match_len = -1; + pCompressor->stats.min_offset = -1; + pCompressor->stats.min_rle1_len = -1; + pCompressor->stats.min_rle2_len = -1; + + if (!nResult) { + pCompressor->intervals = (unsigned int *)malloc(nMaxWindowSize * sizeof(unsigned int)); + + if (pCompressor->intervals) { + pCompressor->pos_data = (unsigned int *)malloc(nMaxWindowSize * sizeof(unsigned int)); + + if (pCompressor->pos_data) { + pCompressor->open_intervals = (unsigned int *)malloc((LCP_AND_TAG_MAX + 1) * sizeof(unsigned int)); + + if (pCompressor->open_intervals) { + pCompressor->arrival = (lzsa_arrival *)malloc(((BLOCK_SIZE + 1) << ARRIVALS_PER_POSITION_SHIFT) * sizeof(lzsa_arrival)); + + if (pCompressor->arrival) { + pCompressor->best_match = (lzsa_match *)malloc(BLOCK_SIZE * sizeof(lzsa_match)); + + if (pCompressor->best_match) { + pCompressor->improved_match = (lzsa_match *)malloc(BLOCK_SIZE * sizeof(lzsa_match)); + + if (pCompressor->improved_match) { + if (pCompressor->format_version == 2) + pCompressor->match = (lzsa_match *)malloc(BLOCK_SIZE * NMATCHES_PER_INDEX_V2 * sizeof(lzsa_match)); + else + pCompressor->match = (lzsa_match *)malloc(BLOCK_SIZE * NMATCHES_PER_INDEX_V1 * sizeof(lzsa_match)); + if (pCompressor->match) { + if (pCompressor->format_version == 2) { + pCompressor->rep_handled_mask = (char*)malloc(NARRIVALS_PER_POSITION_V2_BIG * ((LCP_MAX + 1) / 8) * sizeof(char)); + if (pCompressor->rep_handled_mask) { + pCompressor->first_offset_for_byte = (int*)malloc(65536 * sizeof(int)); + if (pCompressor->first_offset_for_byte) { + pCompressor->next_offset_for_pos = (int*)malloc(BLOCK_SIZE * sizeof(int)); + if (pCompressor->next_offset_for_pos) { + return 0; + } + } + } + } + else { + return 0; + } + } + } + } + } + } + } + } + } + + lzsa_compressor_destroy(pCompressor); + return 100; +} + +/** + * Clean up compression context and free up any associated resources + * + * @param pCompressor compression context to clean up + */ +void lzsa_compressor_destroy(lzsa_compressor *pCompressor) { + divsufsort_destroy(&pCompressor->divsufsort_context); + + if (pCompressor->next_offset_for_pos) { + free(pCompressor->next_offset_for_pos); + pCompressor->next_offset_for_pos = NULL; + } + + if (pCompressor->first_offset_for_byte) { + free(pCompressor->first_offset_for_byte); + pCompressor->first_offset_for_byte = NULL; + } + + if (pCompressor->rep_handled_mask) { + free(pCompressor->rep_handled_mask); + pCompressor->rep_handled_mask = NULL; + } + + if (pCompressor->match) { + free(pCompressor->match); + pCompressor->match = NULL; + } + + if (pCompressor->improved_match) { + free(pCompressor->improved_match); + pCompressor->improved_match = NULL; + } + + if (pCompressor->arrival) { + free(pCompressor->arrival); + pCompressor->arrival = NULL; + } + + if (pCompressor->best_match) { + free(pCompressor->best_match); + pCompressor->best_match = NULL; + } + + if (pCompressor->open_intervals) { + free(pCompressor->open_intervals); + pCompressor->open_intervals = NULL; + } + + if (pCompressor->pos_data) { + free(pCompressor->pos_data); + pCompressor->pos_data = NULL; + } + + if (pCompressor->intervals) { + free(pCompressor->intervals); + pCompressor->intervals = NULL; + } +} + +/** + * Compress one block of data + * + * @param pCompressor compression context + * @param pInWindow pointer to input data window (previously compressed bytes + bytes to compress) + * @param nPreviousBlockSize number of previously compressed bytes (or 0 for none) + * @param nInDataSize number of input bytes to compress + * @param pOutData pointer to output buffer + * @param nMaxOutDataSize maximum size of output buffer, in bytes + * + * @return size of compressed data in output buffer, or -1 if the data is uncompressible + */ +int lzsa_compressor_shrink_block(lzsa_compressor *pCompressor, unsigned char *pInWindow, const int nPreviousBlockSize, const int nInDataSize, unsigned char *pOutData, const int nMaxOutDataSize) { + int nCompressedSize; + + if (pCompressor->flags & LZSA_FLAG_RAW_BACKWARD) { + lzsa_reverse_buffer(pInWindow + nPreviousBlockSize, nInDataSize); + } + + if (lzsa_build_suffix_array(pCompressor, pInWindow, nPreviousBlockSize + nInDataSize)) + nCompressedSize = -1; + else { + if (nPreviousBlockSize) { + lzsa_skip_matches(pCompressor, 0, nPreviousBlockSize); + } + lzsa_find_all_matches(pCompressor, (pCompressor->format_version == 2) ? NMATCHES_PER_INDEX_V2 : NMATCHES_PER_INDEX_V1, nPreviousBlockSize, nPreviousBlockSize + nInDataSize); + + if (pCompressor->format_version == 1) { + nCompressedSize = lzsa_optimize_and_write_block_v1(pCompressor, pInWindow, nPreviousBlockSize, nInDataSize, pOutData, nMaxOutDataSize); + if (nCompressedSize != -1 && (pCompressor->flags & LZSA_FLAG_RAW_BACKWARD)) { + lzsa_reverse_buffer(pOutData, nCompressedSize); + } + } + else if (pCompressor->format_version == 2) { + nCompressedSize = lzsa_optimize_and_write_block_v2(pCompressor, pInWindow, nPreviousBlockSize, nInDataSize, pOutData, nMaxOutDataSize); + if (nCompressedSize != -1 && (pCompressor->flags & LZSA_FLAG_RAW_BACKWARD)) { + lzsa_reverse_buffer(pOutData, nCompressedSize); + } + } + else { + nCompressedSize = -1; + } + } + + if (pCompressor->flags & LZSA_FLAG_RAW_BACKWARD) { + lzsa_reverse_buffer(pInWindow + nPreviousBlockSize, nInDataSize); + } + + return nCompressedSize; +} + +/** + * Get the number of compression commands issued in compressed data blocks + * + * @return number of commands + */ +int lzsa_compressor_get_command_count(lzsa_compressor *pCompressor) { + return pCompressor->num_commands; +} diff --git a/tools/rasm/lzsa-master/src/shrink_context.h b/tools/rasm/lzsa-master/src/shrink_context.h new file mode 100644 index 0000000..54e4671 --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_context.h @@ -0,0 +1,182 @@ +/* + * shrink_context.h - compression context definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _SHRINK_CONTEXT_H +#define _SHRINK_CONTEXT_H + +#include "divsufsort.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define LCP_BITS 14 +#define TAG_BITS 4 +#define LCP_MAX ((1U<<(LCP_BITS - TAG_BITS)) - 1) +#define LCP_AND_TAG_MAX (1U<<(LCP_BITS - 1)) +#define LCP_SHIFT (31-LCP_BITS) +#define LCP_MASK (((1U< + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "shrink_inmem.h" +#include "shrink_context.h" +#include "frame.h" +#include "format.h" +#include "lib.h" + +/** + * Get maximum compressed size of input(source) data + * + * @param nInputSize input(source) size in bytes + * + * @return maximum compressed size + */ +size_t lzsa_get_max_compressed_size_inmem(size_t nInputSize) { + return lzsa_get_header_size() + ((nInputSize + (BLOCK_SIZE - 1)) >> 16) * lzsa_get_frame_size() + nInputSize + lzsa_get_frame_size() /* footer */; +} + +/** + * Compress memory + * + * @param pInputData pointer to input(source) data to compress + * @param pOutBuffer buffer for compressed data + * @param nInputSize input(source) size in bytes + * @param nMaxOutBufferSize maximum capacity of compression buffer + * @param nFlags compression flags (LZSA_FLAG_xxx) + * @param nMinMatchSize minimum match size + * @param nFormatVersion version of format to use (1-2) + * + * @return actual compressed size, or -1 for error + */ +size_t lzsa_compress_inmem(unsigned char *pInputData, unsigned char *pOutBuffer, size_t nInputSize, size_t nMaxOutBufferSize, + const unsigned int nFlags, const int nMinMatchSize, const int nFormatVersion) { + lzsa_compressor compressor; + size_t nOriginalSize = 0; + size_t nCompressedSize = 0L; + int nResult; + int nError = 0; + + nResult = lzsa_compressor_init(&compressor, BLOCK_SIZE * 2, nMinMatchSize, nFormatVersion, nFlags); + if (nResult != 0) { + return -1; + } + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) == 0) { + int nHeaderSize = lzsa_encode_header(pOutBuffer, (int)nMaxOutBufferSize, nFormatVersion); + if (nHeaderSize < 0) + nError = LZSA_ERROR_COMPRESSION; + else { + nCompressedSize += nHeaderSize; + } + } + + int nPreviousBlockSize = 0; + int nNumBlocks = 0; + + while (nOriginalSize < nInputSize && !nError) { + int nInDataSize; + + nInDataSize = (int)(nInputSize - nOriginalSize); + if (nInDataSize > BLOCK_SIZE) + nInDataSize = BLOCK_SIZE; + + if (nInDataSize > 0) { + if ((nFlags & LZSA_FLAG_RAW_BLOCK) != 0 && nNumBlocks) { + nError = LZSA_ERROR_RAW_TOOLARGE; + break; + } + + int nOutDataSize; + int nOutDataEnd = (int)(nMaxOutBufferSize - (lzsa_get_frame_size() + nCompressedSize + lzsa_get_frame_size() /* footer */)); + int nFrameSize = lzsa_get_frame_size(); + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) != 0) { + nFrameSize = 0; + nOutDataEnd = (int)(nMaxOutBufferSize - nCompressedSize); + } + + if (nOutDataEnd > BLOCK_SIZE) + nOutDataEnd = BLOCK_SIZE; + + nOutDataSize = lzsa_compressor_shrink_block(&compressor, pInputData + nOriginalSize - nPreviousBlockSize, nPreviousBlockSize, nInDataSize, pOutBuffer + nFrameSize + nCompressedSize, nOutDataEnd); + if (nOutDataSize >= 0) { + /* Write compressed block */ + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) == 0) { + int nBlockheaderSize = lzsa_encode_compressed_block_frame(pOutBuffer + nCompressedSize, (int)(nMaxOutBufferSize - nCompressedSize), nOutDataSize); + if (nBlockheaderSize < 0) + nError = LZSA_ERROR_COMPRESSION; + else { + nCompressedSize += nBlockheaderSize; + } + } + + if (!nError) { + nOriginalSize += nInDataSize; + nCompressedSize += nOutDataSize; + } + } + else { + /* Write uncompressible, literal block */ + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) != 0) { + nError = LZSA_ERROR_RAW_UNCOMPRESSED; + break; + } + + int nBlockheaderSize = lzsa_encode_uncompressed_block_frame(pOutBuffer + nCompressedSize, (int)(nMaxOutBufferSize - nCompressedSize), nInDataSize); + if (nBlockheaderSize < 0) + nError = LZSA_ERROR_COMPRESSION; + else { + if ((size_t)nInDataSize > (nMaxOutBufferSize - (nCompressedSize + nBlockheaderSize))) + nError = LZSA_ERROR_DST; + else { + memcpy(pOutBuffer + nBlockheaderSize + nCompressedSize, pInputData + nOriginalSize, nInDataSize); + + nOriginalSize += nInDataSize; + nCompressedSize += nBlockheaderSize + nInDataSize; + } + } + } + + nPreviousBlockSize = nInDataSize; + nNumBlocks++; + } + } + + if (!nError) { + int nFooterSize; + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) != 0) { + nFooterSize = 0; + } + else { + nFooterSize = lzsa_encode_footer_frame(pOutBuffer + nCompressedSize, (int)(nMaxOutBufferSize - nCompressedSize)); + if (nFooterSize < 0) + nError = LZSA_ERROR_COMPRESSION; + } + + nCompressedSize += nFooterSize; + } + + lzsa_compressor_destroy(&compressor); + + if (nError) { + return -1; + } + else { + return nCompressedSize; + } +} + diff --git a/tools/rasm/lzsa-master/src/shrink_inmem.h b/tools/rasm/lzsa-master/src/shrink_inmem.h new file mode 100644 index 0000000..2bd8f27 --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_inmem.h @@ -0,0 +1,71 @@ +/* + * shrink_inmem.h - in-memory compression definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _SHRINK_INMEM_H +#define _SHRINK_INMEM_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Get maximum compressed size of input(source) data + * + * @param nInputSize input(source) size in bytes + * + * @return maximum compressed size + */ +size_t lzsa_get_max_compressed_size_inmem(size_t nInputSize); + +/** + * Compress memory + * + * @param pInputData pointer to input(source) data to compress + * @param pOutBuffer buffer for compressed data + * @param nInputSize input(source) size in bytes + * @param nMaxOutBufferSize maximum capacity of compression buffer + * @param nFlags compression flags (LZSA_FLAG_xxx) + * @param nMinMatchSize minimum match size + * @param nFormatVersion version of format to use (1-2) + * + * @return actual compressed size, or -1 for error + */ +size_t lzsa_compress_inmem(unsigned char *pInputData, unsigned char *pOutBuffer, size_t nInputSize, size_t nMaxOutBufferSize, + const unsigned int nFlags, const int nMinMatchSize, const int nFormatVersion); + +#ifdef __cplusplus +} +#endif + +#endif /* _SHRINK_INMEM_H */ diff --git a/tools/rasm/lzsa-master/src/shrink_streaming.c b/tools/rasm/lzsa-master/src/shrink_streaming.c new file mode 100644 index 0000000..2e1cf12 --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_streaming.c @@ -0,0 +1,320 @@ +/* + * shrink_streaming.c - streaming compression implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include "shrink_streaming.h" +#include "format.h" +#include "frame.h" +#include "lib.h" +#ifdef _WIN32 +#include +#else +#include +#endif + +/** + * Delete file + * + * @param pszInFilename name of file to delete + */ +static void lzsa_delete_file(const char *pszInFilename) { +#ifdef _WIN32 + DeleteFileA(pszInFilename); +#else + remove(pszInFilename); +#endif +} + +/*-------------- File API -------------- */ + +/** + * Compress file + * + * @param pszInFilename name of input(source) file to compress + * @param pszOutFilename name of output(compressed) file to generate + * @param pszDictionaryFilename name of dictionary file, or NULL for none + * @param nFlags compression flags (LZSA_FLAG_xxx) + * @param nMinMatchSize minimum match size + * @param nFormatVersion version of format to use (1-2) + * @param progress progress function, called after compressing each block, or NULL for none + * @param pOriginalSize pointer to returned input(source) size, updated when this function is successful + * @param pCompressedSize pointer to returned output(compressed) size, updated when this function is successful + * @param pCommandCount pointer to returned token(compression commands) count, updated when this function is successful + * @param pSafeDist pointer to return safe distance for raw blocks, updated when this function is successful + * @param pStats pointer to compression stats that are filled if this function is successful, or NULL + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +lzsa_status_t lzsa_compress_file(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, const unsigned int nFlags, const int nMinMatchSize, const int nFormatVersion, + void(*progress)(long long nOriginalSize, long long nCompressedSize), long long *pOriginalSize, long long *pCompressedSize, int *pCommandCount, int *pSafeDist, lzsa_stats *pStats) { + lzsa_stream_t inStream, outStream; + void *pDictionaryData = NULL; + int nDictionaryDataSize = 0; + lzsa_status_t nStatus; + + if (lzsa_filestream_open(&inStream, pszInFilename, "rb") < 0) { + return LZSA_ERROR_SRC; + } + + if (lzsa_filestream_open(&outStream, pszOutFilename, "wb") < 0) { + inStream.close(&inStream); + return LZSA_ERROR_DST; + } + + nStatus = lzsa_dictionary_load(pszDictionaryFilename, &pDictionaryData, &nDictionaryDataSize); + + if (nStatus) { + outStream.close(&outStream); + inStream.close(&inStream); + lzsa_delete_file(pszOutFilename); + return nStatus; + } + + nStatus = lzsa_compress_stream(&inStream, &outStream, pDictionaryData, nDictionaryDataSize, nFlags, nMinMatchSize, nFormatVersion, progress, pOriginalSize, pCompressedSize, pCommandCount, pSafeDist, pStats); + + lzsa_dictionary_free(&pDictionaryData); + outStream.close(&outStream); + inStream.close(&inStream); + + if (nStatus) { + lzsa_delete_file(pszOutFilename); + } + + return nStatus; +} + +/*-------------- Streaming API -------------- */ + +/** + * Compress stream + * + * @param pInStream input(source) stream to compress + * @param pOutStream output(compressed) stream to write to + * @param pDictionaryData dictionary contents, or NULL for none + * @param nDictionaryDataSize size of dictionary contents, or 0 + * @param nFlags compression flags (LZSA_FLAG_xxx) + * @param nMinMatchSize minimum match size + * @param nFormatVersion version of format to use (1-2) + * @param progress progress function, called after compressing each block, or NULL for none + * @param pOriginalSize pointer to returned input(source) size, updated when this function is successful + * @param pCompressedSize pointer to returned output(compressed) size, updated when this function is successful + * @param pCommandCount pointer to returned token(compression commands) count, updated when this function is successful + * @param pSafeDist pointer to return safe distance for raw blocks, updated when this function is successful + * @param pStats pointer to compression stats that are filled if this function is successful, or NULL + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +lzsa_status_t lzsa_compress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOutStream, const void *pDictionaryData, int nDictionaryDataSize, + const unsigned int nFlags, const int nMinMatchSize, const int nFormatVersion, + void(*progress)(long long nOriginalSize, long long nCompressedSize), long long *pOriginalSize, long long *pCompressedSize, int *pCommandCount, int *pSafeDist, lzsa_stats *pStats) { + unsigned char *pInData, *pOutData; + lzsa_compressor compressor; + long long nOriginalSize = 0LL, nCompressedSize = 0LL; + int nResult; + unsigned char cFrameData[16]; + int nError = 0; + int nRawPadding = (nFlags & LZSA_FLAG_RAW_BLOCK) ? 8 : 0; + + pInData = (unsigned char*)malloc(BLOCK_SIZE * 2); + if (!pInData) { + return LZSA_ERROR_MEMORY; + } + memset(pInData, 0, BLOCK_SIZE * 2); + + pOutData = (unsigned char*)malloc(BLOCK_SIZE); + if (!pOutData) { + free(pInData); + pInData = NULL; + + return LZSA_ERROR_MEMORY; + } + memset(pOutData, 0, BLOCK_SIZE); + + nResult = lzsa_compressor_init(&compressor, BLOCK_SIZE * 2, nMinMatchSize, nFormatVersion, nFlags); + if (nResult != 0) { + free(pOutData); + pOutData = NULL; + + free(pInData); + pInData = NULL; + + return LZSA_ERROR_MEMORY; + } + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) == 0) { + int nHeaderSize = lzsa_encode_header(cFrameData, 16, nFormatVersion); + if (nHeaderSize < 0) + nError = LZSA_ERROR_COMPRESSION; + else { + if (pOutStream->write(pOutStream, cFrameData, nHeaderSize) != nHeaderSize) + nError = LZSA_ERROR_DST; + nCompressedSize += (long long)nHeaderSize; + } + } + + int nPreviousBlockSize = 0; + int nNumBlocks = 0; + + while (!pInStream->eof(pInStream) && !nError) { + int nInDataSize; + + if (nPreviousBlockSize) { + memcpy(pInData + BLOCK_SIZE - nPreviousBlockSize, pInData + BLOCK_SIZE, nPreviousBlockSize); + } + else if (nDictionaryDataSize && pDictionaryData) { + nPreviousBlockSize = nDictionaryDataSize; + memcpy(pInData + BLOCK_SIZE - nPreviousBlockSize, pDictionaryData, nPreviousBlockSize); + } + + nInDataSize = (int)pInStream->read(pInStream, pInData + BLOCK_SIZE, BLOCK_SIZE); + if (nInDataSize > 0) { + if ((nFlags & LZSA_FLAG_RAW_BLOCK) != 0 && nNumBlocks) { + nError = LZSA_ERROR_RAW_TOOLARGE; + break; + } + nDictionaryDataSize = 0; + + int nOutDataSize; + + nOutDataSize = lzsa_compressor_shrink_block(&compressor, pInData + BLOCK_SIZE - nPreviousBlockSize, nPreviousBlockSize, nInDataSize, pOutData, ((nInDataSize + nRawPadding) >= BLOCK_SIZE) ? BLOCK_SIZE : (nInDataSize + nRawPadding)); + if (nOutDataSize >= 0) { + /* Write compressed block */ + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) == 0) { + int nBlockheaderSize = lzsa_encode_compressed_block_frame(cFrameData, 16, nOutDataSize); + if (nBlockheaderSize < 0) + nError = LZSA_ERROR_COMPRESSION; + else { + nCompressedSize += (long long)nBlockheaderSize; + if (pOutStream->write(pOutStream, cFrameData, nBlockheaderSize) != (size_t)nBlockheaderSize) { + nError = LZSA_ERROR_DST; + } + } + } + + if (!nError) { + if (pOutStream->write(pOutStream, pOutData, (size_t)nOutDataSize) != (size_t)nOutDataSize) { + nError = LZSA_ERROR_DST; + } + else { + nOriginalSize += (long long)nInDataSize; + nCompressedSize += (long long)nOutDataSize; + } + } + } + else { + /* Write uncompressible, literal block */ + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) != 0) { + nError = LZSA_ERROR_RAW_UNCOMPRESSED; + break; + } + + int nBlockheaderSize = lzsa_encode_uncompressed_block_frame(cFrameData, 16, nInDataSize); + if (nBlockheaderSize < 0) + nError = LZSA_ERROR_COMPRESSION; + else { + if (pOutStream->write(pOutStream, cFrameData, nBlockheaderSize) != (size_t)nBlockheaderSize) { + nError = LZSA_ERROR_DST; + } + else { + if (pOutStream->write(pOutStream, pInData + BLOCK_SIZE, (size_t)nInDataSize) != (size_t)nInDataSize) { + nError = LZSA_ERROR_DST; + } + else { + nOriginalSize += (long long)nInDataSize; + nCompressedSize += (long long)nBlockheaderSize + (long long)nInDataSize; + } + } + } + } + + nPreviousBlockSize = nInDataSize; + nNumBlocks++; + } + + if (!nError && !pInStream->eof(pInStream)) { + if (progress) + progress(nOriginalSize, nCompressedSize); + } + } + + if (!nError) { + int nFooterSize; + + if ((nFlags & LZSA_FLAG_RAW_BLOCK) != 0) { + nFooterSize = 0; + } + else { + nFooterSize = lzsa_encode_footer_frame(cFrameData, 16); + if (nFooterSize < 0) + nError = LZSA_ERROR_COMPRESSION; + } + + if (pOutStream->write(pOutStream, cFrameData, nFooterSize) != nFooterSize) + nError = LZSA_ERROR_DST; + nCompressedSize += (long long)nFooterSize; + } + + if (progress) + progress(nOriginalSize, nCompressedSize); + + int nCommandCount = lzsa_compressor_get_command_count(&compressor); + int nSafeDist = compressor.safe_dist; + + if (pStats) + *pStats = compressor.stats; + + lzsa_compressor_destroy(&compressor); + + free(pOutData); + pOutData = NULL; + + free(pInData); + pInData = NULL; + + if (nError) { + return nError; + } + else { + if (pOriginalSize) + *pOriginalSize = nOriginalSize; + if (pCompressedSize) + *pCompressedSize = nCompressedSize; + if (pCommandCount) + *pCommandCount = nCommandCount; + if (pSafeDist) + *pSafeDist = nSafeDist; + return LZSA_OK; + } +} diff --git a/tools/rasm/lzsa-master/src/shrink_streaming.h b/tools/rasm/lzsa-master/src/shrink_streaming.h new file mode 100644 index 0000000..0920edf --- /dev/null +++ b/tools/rasm/lzsa-master/src/shrink_streaming.h @@ -0,0 +1,99 @@ +/* + * shrink_streaming.h - streaming compression definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _SHRINK_STREAMING_H +#define _SHRINK_STREAMING_H + +#include "stream.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Forward declaration */ +typedef enum _lzsa_status_t lzsa_status_t; +typedef struct _lzsa_stats lzsa_stats; + +/*-------------- File API -------------- */ + +/** + * Compress file + * + * @param pszInFilename name of input(source) file to compress + * @param pszOutFilename name of output(compressed) file to generate + * @param pszDictionaryFilename name of dictionary file, or NULL for none + * @param nFlags compression flags (LZSA_FLAG_xxx) + * @param nMinMatchSize minimum match size + * @param nFormatVersion version of format to use (1-2) + * @param progress progress function, called after compressing each block, or NULL for none + * @param pOriginalSize pointer to returned input(source) size, updated when this function is successful + * @param pCompressedSize pointer to returned output(compressed) size, updated when this function is successful + * @param pCommandCount pointer to returned token(compression commands) count, updated when this function is successful + * @param pSafeDist pointer to return safe distance for raw blocks, updated when this function is successful + * @param pStats pointer to compression stats that are filled if this function is successful, or NULL + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +lzsa_status_t lzsa_compress_file(const char *pszInFilename, const char *pszOutFilename, const char *pszDictionaryFilename, + const unsigned int nFlags, const int nMinMatchSize, const int nFormatVersion, + void(*progress)(long long nOriginalSize, long long nCompressedSize), long long *pOriginalSize, long long *pCompressedSize, int *pCommandCount, int *pSafeDist, lzsa_stats *pStats); + +/*-------------- Streaming API -------------- */ + +/** + * Compress stream + * + * @param pInStream input(source) stream to compress + * @param pOutStream output(compressed) stream to write to + * @param pDictionaryData dictionary contents, or NULL for none + * @param nDictionaryDataSize size of dictionary contents, or 0 + * @param nFlags compression flags (LZSA_FLAG_xxx) + * @param nMinMatchSize minimum match size + * @param nFormatVersion version of format to use (1-2) + * @param progress progress function, called after compressing each block, or NULL for none + * @param pOriginalSize pointer to returned input(source) size, updated when this function is successful + * @param pCompressedSize pointer to returned output(compressed) size, updated when this function is successful + * @param pCommandCount pointer to returned token(compression commands) count, updated when this function is successful + * @param pSafeDist pointer to return safe distance for raw blocks, updated when this function is successful + * @param pStats pointer to compression stats that are filled if this function is successful, or NULL + * + * @return LZSA_OK for success, or an error value from lzsa_status_t + */ +lzsa_status_t lzsa_compress_stream(lzsa_stream_t *pInStream, lzsa_stream_t *pOutStream, const void *pDictionaryData, int nDictionaryDataSize, + const unsigned int nFlags, const int nMinMatchSize, const int nFormatVersion, + void(*progress)(long long nOriginalSize, long long nCompressedSize), long long *pOriginalSize, long long *pCompressedSize, int *pCommandCount, int *pSafeDist, lzsa_stats *pStats); + +#ifdef __cplusplus +} +#endif + +#endif /* _SHRINK_STREAMING_H */ diff --git a/tools/rasm/lzsa-master/src/stream.c b/tools/rasm/lzsa-master/src/stream.c new file mode 100644 index 0000000..8937487 --- /dev/null +++ b/tools/rasm/lzsa-master/src/stream.c @@ -0,0 +1,111 @@ +/* + * stream.c - streaming I/O implementation + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#include +#include +#include +#include "stream.h" + +/** + * Close file stream + * + * @param stream stream + */ +static void lzsa_filestream_close(lzsa_stream_t *stream) { + if (stream->obj) { + fclose((FILE*)stream->obj); + stream->obj = NULL; + stream->read = NULL; + stream->write = NULL; + stream->eof = NULL; + stream->close = NULL; + } +} + +/** + * Read from file stream + * + * @param stream stream + * @param ptr buffer to read into + * @param size number of bytes to read + * + * @return number of bytes read + */ +static size_t lzsa_filestream_read(lzsa_stream_t *stream, void *ptr, size_t size) { + return fread(ptr, 1, size, (FILE*)stream->obj); +} + +/** + * Write to file stream + * + * @param stream stream + * @param ptr buffer to write from + * @param size number of bytes to write + * + * @return number of bytes written + */ +static size_t lzsa_filestream_write(lzsa_stream_t *stream, void *ptr, size_t size) { + return fwrite(ptr, 1, size, (FILE*)stream->obj); +} + +/** + * Check if file stream has reached the end of the data + * + * @param stream stream + * + * @return nonzero if the end of the data has been reached, 0 if there is more data + */ +static int lzsa_filestream_eof(lzsa_stream_t *stream) { + return feof((FILE*)stream->obj); +} + +/** + * Open file and create an I/O stream from it + * + * @param stream stream to fill out + * @param pszInFilename filename + * @param pszMode open mode, as with fopen() + * + * @return 0 for success, nonzero for failure + */ +int lzsa_filestream_open(lzsa_stream_t *stream, const char *pszInFilename, const char *pszMode) { + stream->obj = (void*)fopen(pszInFilename, pszMode); + if (stream->obj) { + stream->read = lzsa_filestream_read; + stream->write = lzsa_filestream_write; + stream->eof = lzsa_filestream_eof; + stream->close = lzsa_filestream_close; + return 0; + } + else + return -1; +} diff --git a/tools/rasm/lzsa-master/src/stream.h b/tools/rasm/lzsa-master/src/stream.h new file mode 100644 index 0000000..a8b7922 --- /dev/null +++ b/tools/rasm/lzsa-master/src/stream.h @@ -0,0 +1,103 @@ +/* + * stream.h - streaming I/O definitions + * + * Copyright (C) 2019 Emmanuel Marty + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + */ + +/* + * Uses the libdivsufsort library Copyright (c) 2003-2008 Yuta Mori + * + * Inspired by LZ4 by Yann Collet. https://github.com/lz4/lz4 + * With help, ideas, optimizations and speed measurements by spke + * With ideas from Lizard by Przemyslaw Skibinski and Yann Collet. https://github.com/inikep/lizard + * Also with ideas from smallz4 by Stephan Brumme. https://create.stephan-brumme.com/smallz4/ + * + */ + +#ifndef _STREAM_H +#define _STREAM_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Forward declaration */ +typedef struct _lzsa_stream_t lzsa_stream_t; + +/* I/O stream */ +typedef struct _lzsa_stream_t { + /** Opaque stream-specific pointer */ + void *obj; + + /** + * Read from stream + * + * @param stream stream + * @param ptr buffer to read into + * @param size number of bytes to read + * + * @return number of bytes read + */ + size_t(*read)(lzsa_stream_t *stream, void *ptr, size_t size); + + /** + * Write to stream + * + * @param stream stream + * @param ptr buffer to write from + * @param size number of bytes to write + * + * @return number of bytes written + */ + size_t(*write)(lzsa_stream_t *stream, void *ptr, size_t size); + + + /** + * Check if stream has reached the end of the data + * + * @param stream stream + * + * @return nonzero if the end of the data has been reached, 0 if there is more data + */ + int(*eof)(lzsa_stream_t *stream); + + /** + * Close stream + * + * @param stream stream + */ + void(*close)(lzsa_stream_t *stream); +} lzsa_stream_t; + +/** + * Open file and create an I/O stream from it + * + * @param stream stream to fill out + * @param pszInFilename filename + * @param pszMode open mode, as with fopen() + * + * @return 0 for success, nonzero for failure + */ +int lzsa_filestream_open(lzsa_stream_t *stream, const char *pszInFilename, const char *pszMode); + +#ifdef __cplusplus +} +#endif + +#endif /* _STREAM_H */ -- cgit v1.2.3