diff options
-rw-r--r-- | src/lzss/listpolyfill.nim | 42 | ||||
-rw-r--r-- | src/lzss/lzsschain.nim | 17 | ||||
-rw-r--r-- | src/lzss/lzssencoder.nim | 16 | ||||
-rw-r--r-- | src/lzss/matchtable.nim | 14 | ||||
-rw-r--r-- | src/lzsshuffman/lzsshuffmandecoder.nim | 7 | ||||
-rw-r--r-- | src/lzsshuffman/lzsshuffmanencoder.nim | 5 | ||||
-rw-r--r-- | src/lzsshuffman/lzsshuffmanstats.nim | 2 | ||||
-rw-r--r-- | tests/tlzss.nim | 51 | ||||
-rw-r--r-- | tests/tlzsshuffman.nim | 8 |
9 files changed, 47 insertions, 115 deletions
diff --git a/src/lzss/listpolyfill.nim b/src/lzss/listpolyfill.nim deleted file mode 100644 index 00b30ee..0000000 --- a/src/lzss/listpolyfill.nim +++ /dev/null | |||
@@ -1,42 +0,0 @@ | |||
1 | # gzip-like LZSS compressor | ||
2 | # Copyright (C) 2018 Pacien TRAN-GIRARD | ||
3 | # | ||
4 | # This program is free software: you can redistribute it and/or modify | ||
5 | # it under the terms of the GNU Affero General Public License as | ||
6 | # published by the Free Software Foundation, either version 3 of the | ||
7 | # License, or (at your option) any later version. | ||
8 | # | ||
9 | # This program is distributed in the hope that it will be useful, | ||
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
12 | # GNU Affero General Public License for more details. | ||
13 | # | ||
14 | # You should have received a copy of the GNU Affero General Public License | ||
15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. | ||
16 | |||
17 | import lists | ||
18 | |||
19 | # https://github.com/nim-lang/Nim/pull/9805 | ||
20 | |||
21 | proc prepend*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) = | ||
22 | ## prepends a node to `L`. Efficiency: O(1). | ||
23 | n.next = L.head | ||
24 | L.head = n | ||
25 | if L.tail == nil: L.tail = n | ||
26 | |||
27 | proc prepend*[T](L: var SinglyLinkedList[T], value: T) = | ||
28 | ## prepends a node to `L`. Efficiency: O(1). | ||
29 | listpolyfill.prepend(L, newSinglyLinkedNode(value)) | ||
30 | |||
31 | proc append*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) = | ||
32 | ## appends a node `n` to `L`. Efficiency: O(1). | ||
33 | n.next = nil | ||
34 | if L.tail != nil: | ||
35 | assert(L.tail.next == nil) | ||
36 | L.tail.next = n | ||
37 | L.tail = n | ||
38 | if L.head == nil: L.head = n | ||
39 | |||
40 | proc append*[T](L: var SinglyLinkedList[T], value: T) = | ||
41 | ## appends a value to `L`. Efficiency: O(1). | ||
42 | append(L, newSinglyLinkedNode(value)) | ||
diff --git a/src/lzss/lzsschain.nim b/src/lzss/lzsschain.nim index 8ebcb1a..ab03655 100644 --- a/src/lzss/lzsschain.nim +++ b/src/lzss/lzsschain.nim | |||
@@ -14,26 +14,23 @@ | |||
14 | # You should have received a copy of the GNU Affero General Public License | 14 | # You should have received a copy of the GNU Affero General Public License |
15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. | 15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. |
16 | 16 | ||
17 | import lists, tables, sugar | 17 | import tables, sugar |
18 | import ../bitio/integers | 18 | import ../bitio/integers |
19 | import listpolyfill, lzssnode | 19 | import lzssnode |
20 | 20 | ||
21 | const maxChainByteLength = 32_000 * wordBitLength | 21 | const maxChainByteLength = 32_000 |
22 | 22 | ||
23 | type LzssChain* = | 23 | type LzssChain* = seq[LzssNode] |
24 | SinglyLinkedList[LzssNode] | ||
25 | 24 | ||
26 | proc lzssChain*(): LzssChain = | 25 | proc lzssChain*(): LzssChain = |
27 | initSinglyLinkedList[LzssNode]() | 26 | newSeq[LzssNode]() |
28 | 27 | ||
29 | proc lzssChain*(chainArray: openArray[LzssNode]): LzssChain = | 28 | proc lzssChain*(chainArray: openArray[LzssNode]): LzssChain = |
30 | var chain = lzssChain() | 29 | @chainArray |
31 | for node in chainArray: chain.append(node) | ||
32 | chain | ||
33 | 30 | ||
34 | proc decode*(lzssChain: LzssChain): seq[uint8] = | 31 | proc decode*(lzssChain: LzssChain): seq[uint8] = |
35 | result = newSeqOfCap[uint8](maxChainByteLength) | 32 | result = newSeqOfCap[uint8](maxChainByteLength) |
36 | for node in lzssChain.items: | 33 | for node in lzssChain: |
37 | case node.kind: | 34 | case node.kind: |
38 | of character: | 35 | of character: |
39 | result.add(node.character) | 36 | result.add(node.character) |
diff --git a/src/lzss/lzssencoder.nim b/src/lzss/lzssencoder.nim index 82fbe7b..36e0c7e 100644 --- a/src/lzss/lzssencoder.nim +++ b/src/lzss/lzssencoder.nim | |||
@@ -14,20 +14,18 @@ | |||
14 | # You should have received a copy of the GNU Affero General Public License | 14 | # You should have received a copy of the GNU Affero General Public License |
15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. | 15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. |
16 | 16 | ||
17 | import lists | 17 | import matchtable, lzssnode, lzsschain |
18 | import listpolyfill, matchtable, lzssnode, lzsschain | ||
19 | 18 | ||
20 | const matchGroupLength* = 3 | 19 | const matchGroupLength* = 3 |
21 | const maxRefByteLength = high(uint8).int + matchGroupLength | 20 | const maxRefByteLength = high(uint8).int + matchGroupLength |
22 | let emptySinglyLinkedList = initSinglyLinkedList[int]() | ||
23 | 21 | ||
24 | proc commonPrefixLength*(a, b: openArray[uint8], skipFirst, maxLength: int): int = | 22 | proc commonPrefixLength*(a, b: openArray[uint8], skipFirst, maxLength: int): int = |
25 | result = skipFirst | 23 | result = skipFirst |
26 | let maxPrefixLength = min(min(a.len, b.len), maxLength) | 24 | let maxPrefixLength = min(min(a.len, b.len), maxLength) |
27 | while result < maxPrefixLength and a[result] == b[result]: result += 1 | 25 | while result < maxPrefixLength and a[result] == b[result]: result += 1 |
28 | 26 | ||
29 | proc longestPrefix*(candidatePos: SinglyLinkedList[int], searchBuf, lookAheadBuf: openArray[uint8]): tuple[length, pos: int] = | 27 | proc longestPrefix*(candidatePos: openArray[int], searchBuf, lookAheadBuf: openArray[uint8]): tuple[length, pos: int] = |
30 | for startIndex in candidatePos.items: | 28 | for startIndex in candidatePos: |
31 | let prefixLength = commonPrefixLength( | 29 | let prefixLength = commonPrefixLength( |
32 | searchBuf.toOpenArray(startIndex, searchBuf.len - 1), lookAheadBuf, matchGroupLength, maxRefByteLength) | 30 | searchBuf.toOpenArray(startIndex, searchBuf.len - 1), lookAheadBuf, matchGroupLength, maxRefByteLength) |
33 | if prefixLength > result.length: result = (prefixLength, startIndex) | 31 | if prefixLength > result.length: result = (prefixLength, startIndex) |
@@ -39,20 +37,20 @@ proc addGroups*(matchTable: MatchTable[seq[uint8], int], buffer: openArray[uint8 | |||
39 | matchTable.addMatch(group, cursor) | 37 | matchTable.addMatch(group, cursor) |
40 | 38 | ||
41 | proc lzssEncode*(buf: openArray[uint8]): LzssChain = | 39 | proc lzssEncode*(buf: openArray[uint8]): LzssChain = |
42 | result = initSinglyLinkedList[LzssNode]() | 40 | result = newSeqOfCap[LzssNode](buf.len) |
43 | let matchTable = initMatchTable(seq[uint8], int) | 41 | let matchTable = initMatchTable(seq[uint8], int) |
44 | var cursor = 0 | 42 | var cursor = 0 |
45 | while cursor < buf.len() - matchGroupLength: | 43 | while cursor < buf.len() - matchGroupLength: |
46 | let matches = matchTable.matchList(buf[cursor..<(cursor + matchGroupLength)]) | 44 | let matches = matchTable.matchList(buf[cursor..<(cursor + matchGroupLength)]) |
47 | let prefix = matches.longestPrefix(buf.toOpenArray(0, cursor - 1), buf.toOpenArray(cursor, buf.len - 1)) | 45 | let prefix = matches.longestPrefix(buf.toOpenArray(0, cursor - 1), buf.toOpenArray(cursor, buf.len - 1)) |
48 | if prefix.length > 0: | 46 | if prefix.length > 0: |
49 | result.append(lzssReference(prefix.length, cursor - prefix.pos)) | 47 | result.add(lzssReference(prefix.length, cursor - prefix.pos)) |
50 | cursor += prefix.length | 48 | cursor += prefix.length |
51 | else: | 49 | else: |
52 | result.append(lzssCharacter(buf[cursor])) | 50 | result.add(lzssCharacter(buf[cursor])) |
53 | cursor += 1 | 51 | cursor += 1 |
54 | if cursor - prefix.length >= matchGroupLength: | 52 | if cursor - prefix.length >= matchGroupLength: |
55 | matchTable.addGroups(buf, cursor - prefix.length - matchGroupLength, cursor) | 53 | matchTable.addGroups(buf, cursor - prefix.length - matchGroupLength, cursor) |
56 | while cursor < buf.len: | 54 | while cursor < buf.len: |
57 | result.append(lzssCharacter(buf[cursor])) | 55 | result.add(lzssCharacter(buf[cursor])) |
58 | cursor += 1 | 56 | cursor += 1 |
diff --git a/src/lzss/matchtable.nim b/src/lzss/matchtable.nim index b17ce68..94fe208 100644 --- a/src/lzss/matchtable.nim +++ b/src/lzss/matchtable.nim | |||
@@ -14,19 +14,17 @@ | |||
14 | # You should have received a copy of the GNU Affero General Public License | 14 | # You should have received a copy of the GNU Affero General Public License |
15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. | 15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. |
16 | 16 | ||
17 | import tables, lists | 17 | import tables |
18 | import listpolyfill | ||
19 | 18 | ||
20 | type MatchTable*[K, V] = | 19 | type MatchTable*[K, V] = TableRef[K, seq[V]] |
21 | TableRef[K, SinglyLinkedList[V]] | ||
22 | 20 | ||
23 | proc initMatchTable*[K, V](keyType: typedesc[K], valueType: typedesc[V]): MatchTable[K, V] = | 21 | proc initMatchTable*[K, V](keyType: typedesc[K], valueType: typedesc[V]): MatchTable[K, V] = |
24 | newTable[K, SinglyLinkedList[V]]() | 22 | newTable[K, seq[V]]() |
25 | 23 | ||
26 | proc matchList*[K, V](matchTable: MatchTable[K, V], pattern: K): SinglyLinkedList[V] = | 24 | proc matchList*[K, V](matchTable: MatchTable[K, V], pattern: K): seq[V] = |
27 | matchTable.getOrDefault(pattern, initSinglyLinkedList[V]()) | 25 | matchTable.getOrDefault(pattern, newSeq[V]()) |
28 | 26 | ||
29 | proc addMatch*[K, V](matchTable: MatchTable[K, V], pattern: K, value: V) = | 27 | proc addMatch*[K, V](matchTable: MatchTable[K, V], pattern: K, value: V) = |
30 | var matchList = matchTable.matchList(pattern) | 28 | var matchList = matchTable.matchList(pattern) |
31 | listpolyfill.prepend(matchList, value) | 29 | matchList.insert(value) |
32 | matchTable[pattern] = matchList | 30 | matchTable[pattern] = matchList |
diff --git a/src/lzsshuffman/lzsshuffmandecoder.nim b/src/lzsshuffman/lzsshuffmandecoder.nim index cd71914..a307774 100644 --- a/src/lzsshuffman/lzsshuffmandecoder.nim +++ b/src/lzsshuffman/lzsshuffmandecoder.nim | |||
@@ -14,9 +14,8 @@ | |||
14 | # You should have received a copy of the GNU Affero General Public License | 14 | # You should have received a copy of the GNU Affero General Public License |
15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. | 15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. |
16 | 16 | ||
17 | import lists | ||
18 | import ../bitio/bitreader | 17 | import ../bitio/bitreader |
19 | import ../lzss/listpolyfill, ../lzss/lzssnode, ../lzss/lzsschain | 18 | import ../lzss/lzssnode, ../lzss/lzsschain |
20 | import ../huffman/huffmantree, ../huffman/huffmandecoder | 19 | import ../huffman/huffmantree, ../huffman/huffmandecoder |
21 | import lzsshuffmansymbol | 20 | import lzsshuffmansymbol |
22 | 21 | ||
@@ -26,9 +25,9 @@ proc readChain*(bitReader: BitReader, symbolDecoder, positionDecoder: HuffmanDec | |||
26 | while not symbol.isEndMarker(): | 25 | while not symbol.isEndMarker(): |
27 | if byteCursor > maxDataByteLength: raise newException(IOError, "lzss block too long") | 26 | if byteCursor > maxDataByteLength: raise newException(IOError, "lzss block too long") |
28 | if symbol.isCharacter(): | 27 | if symbol.isCharacter(): |
29 | chain.append(lzssCharacter(symbol.uint8)) | 28 | chain.add(lzssCharacter(symbol.uint8)) |
30 | else: | 29 | else: |
31 | let position = positionDecoder.decode(bitReader) | 30 | let position = positionDecoder.decode(bitReader) |
32 | chain.append(unpackLzssReference(symbol, position)) | 31 | chain.add(unpackLzssReference(symbol, position)) |
33 | (symbol, byteCursor) = (symbolDecoder.decode(bitReader).Symbol, byteCursor + 1) | 32 | (symbol, byteCursor) = (symbolDecoder.decode(bitReader).Symbol, byteCursor + 1) |
34 | chain | 33 | chain |
diff --git a/src/lzsshuffman/lzsshuffmanencoder.nim b/src/lzsshuffman/lzsshuffmanencoder.nim index ea89f85..205f464 100644 --- a/src/lzsshuffman/lzsshuffmanencoder.nim +++ b/src/lzsshuffman/lzsshuffmanencoder.nim | |||
@@ -14,9 +14,8 @@ | |||
14 | # You should have received a copy of the GNU Affero General Public License | 14 | # You should have received a copy of the GNU Affero General Public License |
15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. | 15 | # along with this program. If not, see <https://www.gnu.org/licenses/>. |
16 | 16 | ||
17 | import lists | ||
18 | import ../bitio/bitwriter | 17 | import ../bitio/bitwriter |
19 | import ../lzss/listpolyfill, ../lzss/lzssnode, ../lzss/lzsschain, ../lzss/lzssencoder | 18 | import ../lzss/lzssnode, ../lzss/lzsschain, ../lzss/lzssencoder |
20 | import ../huffman/huffmantree, ../huffman/huffmantreebuilder, ../huffman/huffmanencoder | 19 | import ../huffman/huffmantree, ../huffman/huffmantreebuilder, ../huffman/huffmanencoder |
21 |