aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/lzss/listpolyfill.nim42
-rw-r--r--src/lzss/lzsschain.nim17
-rw-r--r--src/lzss/lzssencoder.nim16
-rw-r--r--src/lzss/matchtable.nim14
-rw-r--r--src/lzsshuffman/lzsshuffmandecoder.nim7
-rw-r--r--src/lzsshuffman/lzsshuffmanencoder.nim5
-rw-r--r--src/lzsshuffman/lzsshuffmanstats.nim2
-rw-r--r--tests/tlzss.nim51
-rw-r--r--tests/tlzsshuffman.nim8
9 files changed, 47 insertions, 115 deletions
diff --git a/src/lzss/listpolyfill.nim b/src/lzss/listpolyfill.nim
deleted file mode 100644
index 00b30ee..0000000
--- a/src/lzss/listpolyfill.nim
+++ /dev/null
@@ -1,42 +0,0 @@
1# gzip-like LZSS compressor
2# Copyright (C) 2018 Pacien TRAN-GIRARD
3#
4# This program is free software: you can redistribute it and/or modify
5# it under the terms of the GNU Affero General Public License as
6# published by the Free Software Foundation, either version 3 of the
7# License, or (at your option) any later version.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Affero General Public License for more details.
13#
14# You should have received a copy of the GNU Affero General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16
17import lists
18
19# https://github.com/nim-lang/Nim/pull/9805
20
21proc prepend*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) =
22 ## prepends a node to `L`. Efficiency: O(1).
23 n.next = L.head
24 L.head = n
25 if L.tail == nil: L.tail = n
26
27proc prepend*[T](L: var SinglyLinkedList[T], value: T) =
28 ## prepends a node to `L`. Efficiency: O(1).
29 listpolyfill.prepend(L, newSinglyLinkedNode(value))
30
31proc append*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) =
32 ## appends a node `n` to `L`. Efficiency: O(1).
33 n.next = nil
34 if L.tail != nil:
35 assert(L.tail.next == nil)
36 L.tail.next = n
37 L.tail = n
38 if L.head == nil: L.head = n
39
40proc append*[T](L: var SinglyLinkedList[T], value: T) =
41 ## appends a value to `L`. Efficiency: O(1).
42 append(L, newSinglyLinkedNode(value))
diff --git a/src/lzss/lzsschain.nim b/src/lzss/lzsschain.nim
index 8ebcb1a..ab03655 100644
--- a/src/lzss/lzsschain.nim
+++ b/src/lzss/lzsschain.nim
@@ -14,26 +14,23 @@
14# You should have received a copy of the GNU Affero General Public License 14# You should have received a copy of the GNU Affero General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>. 15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16 16
17import lists, tables, sugar 17import tables, sugar
18import ../bitio/integers 18import ../bitio/integers
19import listpolyfill, lzssnode 19import lzssnode
20 20
21const maxChainByteLength = 32_000 * wordBitLength 21const maxChainByteLength = 32_000
22 22
23type LzssChain* = 23type LzssChain* = seq[LzssNode]
24 SinglyLinkedList[LzssNode]
25 24
26proc lzssChain*(): LzssChain = 25proc lzssChain*(): LzssChain =
27 initSinglyLinkedList[LzssNode]() 26 newSeq[LzssNode]()
28 27
29proc lzssChain*(chainArray: openArray[LzssNode]): LzssChain = 28proc lzssChain*(chainArray: openArray[LzssNode]): LzssChain =
30 var chain = lzssChain() 29 @chainArray
31 for node in chainArray: chain.append(node)
32 chain
33 30
34proc decode*(lzssChain: LzssChain): seq[uint8] = 31proc decode*(lzssChain: LzssChain): seq[uint8] =
35 result = newSeqOfCap[uint8](maxChainByteLength) 32 result = newSeqOfCap[uint8](maxChainByteLength)
36 for node in lzssChain.items: 33 for node in lzssChain:
37 case node.kind: 34 case node.kind:
38 of character: 35 of character:
39 result.add(node.character) 36 result.add(node.character)
diff --git a/src/lzss/lzssencoder.nim b/src/lzss/lzssencoder.nim
index 82fbe7b..36e0c7e 100644
--- a/src/lzss/lzssencoder.nim
+++ b/src/lzss/lzssencoder.nim
@@ -14,20 +14,18 @@
14# You should have received a copy of the GNU Affero General Public License 14# You should have received a copy of the GNU Affero General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>. 15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16 16
17import lists 17import matchtable, lzssnode, lzsschain
18import listpolyfill, matchtable, lzssnode, lzsschain
19 18
20const matchGroupLength* = 3 19const matchGroupLength* = 3
21const maxRefByteLength = high(uint8).int + matchGroupLength 20const maxRefByteLength = high(uint8).int + matchGroupLength
22let emptySinglyLinkedList = initSinglyLinkedList[int]()
23 21
24proc commonPrefixLength*(a, b: openArray[uint8], skipFirst, maxLength: int): int = 22proc commonPrefixLength*(a, b: openArray[uint8], skipFirst, maxLength: int): int =
25 result = skipFirst 23 result = skipFirst
26 let maxPrefixLength = min(min(a.len, b.len), maxLength) 24 let maxPrefixLength = min(min(a.len, b.len), maxLength)
27 while result < maxPrefixLength and a[result] == b[result]: result += 1 25 while result < maxPrefixLength and a[result] == b[result]: result += 1
28 26
29proc longestPrefix*(candidatePos: SinglyLinkedList[int], searchBuf, lookAheadBuf: openArray[uint8]): tuple[length, pos: int] = 27proc longestPrefix*(candidatePos: openArray[int], searchBuf, lookAheadBuf: openArray[uint8]): tuple[length, pos: int] =
30 for startIndex in candidatePos.items: 28 for startIndex in candidatePos:
31 let prefixLength = commonPrefixLength( 29 let prefixLength = commonPrefixLength(
32 searchBuf.toOpenArray(startIndex, searchBuf.len - 1), lookAheadBuf, matchGroupLength, maxRefByteLength) 30 searchBuf.toOpenArray(startIndex, searchBuf.len - 1), lookAheadBuf, matchGroupLength, maxRefByteLength)
33 if prefixLength > result.length: result = (prefixLength, startIndex) 31 if prefixLength > result.length: result = (prefixLength, startIndex)
@@ -39,20 +37,20 @@ proc addGroups*(matchTable: MatchTable[seq[uint8], int], buffer: openArray[uint8
39 matchTable.addMatch(group, cursor) 37 matchTable.addMatch(group, cursor)
40 38
41proc lzssEncode*(buf: openArray[uint8]): LzssChain = 39proc lzssEncode*(buf: openArray[uint8]): LzssChain =
42 result = initSinglyLinkedList[LzssNode]() 40 result = newSeqOfCap[LzssNode](buf.len)
43 let matchTable = initMatchTable(seq[uint8], int) 41 let matchTable = initMatchTable(seq[uint8], int)
44 var cursor = 0 42 var cursor = 0
45 while cursor < buf.len() - matchGroupLength: 43 while cursor < buf.len() - matchGroupLength:
46 let matches = matchTable.matchList(buf[cursor..<(cursor + matchGroupLength)]) 44 let matches = matchTable.matchList(buf[cursor..<(cursor + matchGroupLength)])
47 let prefix = matches.longestPrefix(buf.toOpenArray(0, cursor - 1), buf.toOpenArray(cursor, buf.len - 1)) 45 let prefix = matches.longestPrefix(buf.toOpenArray(0, cursor - 1), buf.toOpenArray(cursor, buf.len - 1))
48 if prefix.length > 0: 46 if prefix.length > 0:
49 result.append(lzssReference(prefix.length, cursor - prefix.pos)) 47 result.add(lzssReference(prefix.length, cursor - prefix.pos))
50 cursor += prefix.length 48 cursor += prefix.length
51 else: 49 else:
52 result.append(lzssCharacter(buf[cursor])) 50 result.add(lzssCharacter(buf[cursor]))
53 cursor += 1 51 cursor += 1
54 if cursor - prefix.length >= matchGroupLength: 52 if cursor - prefix.length >= matchGroupLength:
55 matchTable.addGroups(buf, cursor - prefix.length - matchGroupLength, cursor) 53 matchTable.addGroups(buf, cursor - prefix.length - matchGroupLength, cursor)
56 while cursor < buf.len: 54 while cursor < buf.len:
57 result.append(lzssCharacter(buf[cursor])) 55 result.add(lzssCharacter(buf[cursor]))
58 cursor += 1 56 cursor += 1
diff --git a/src/lzss/matchtable.nim b/src/lzss/matchtable.nim
index b17ce68..94fe208 100644
--- a/src/lzss/matchtable.nim
+++ b/src/lzss/matchtable.nim
@@ -14,19 +14,17 @@
14# You should have received a copy of the GNU Affero General Public License 14# You should have received a copy of the GNU Affero General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>. 15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16 16
17import tables, lists 17import tables
18import listpolyfill
19 18
20type MatchTable*[K, V] = 19type MatchTable*[K, V] = TableRef[K, seq[V]]
21 TableRef[K, SinglyLinkedList[V]]
22 20
23proc initMatchTable*[K, V](keyType: typedesc[K], valueType: typedesc[V]): MatchTable[K, V] = 21proc initMatchTable*[K, V](keyType: typedesc[K], valueType: typedesc[V]): MatchTable[K, V] =
24 newTable[K, SinglyLinkedList[V]]() 22 newTable[K, seq[V]]()
25 23
26proc matchList*[K, V](matchTable: MatchTable[K, V], pattern: K): SinglyLinkedList[V] = 24proc matchList*[K, V](matchTable: MatchTable[K, V], pattern: K): seq[V] =
27 matchTable.getOrDefault(pattern, initSinglyLinkedList[V]()) 25 matchTable.getOrDefault(pattern, newSeq[V]())
28 26
29proc addMatch*[K, V](matchTable: MatchTable[K, V], pattern: K, value: V) = 27proc addMatch*[K, V](matchTable: MatchTable[K, V], pattern: K, value: V) =
30 var matchList = matchTable.matchList(pattern) 28 var matchList = matchTable.matchList(pattern)
31 listpolyfill.prepend(matchList, value) 29 matchList.insert(value)
32 matchTable[pattern] = matchList 30 matchTable[pattern] = matchList
diff --git a/src/lzsshuffman/lzsshuffmandecoder.nim b/src/lzsshuffman/lzsshuffmandecoder.nim
index cd71914..a307774 100644
--- a/src/lzsshuffman/lzsshuffmandecoder.nim
+++ b/src/lzsshuffman/lzsshuffmandecoder.nim
@@ -14,9 +14,8 @@
14# You should have received a copy of the GNU Affero General Public License 14# You should have received a copy of the GNU Affero General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>. 15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16 16
17import lists
18import ../bitio/bitreader 17import ../bitio/bitreader
19import ../lzss/listpolyfill, ../lzss/lzssnode, ../lzss/lzsschain 18import ../lzss/lzssnode, ../lzss/lzsschain
20import ../huffman/huffmantree, ../huffman/huffmandecoder 19import ../huffman/huffmantree, ../huffman/huffmandecoder
21import lzsshuffmansymbol 20import lzsshuffmansymbol
22 21
@@ -26,9 +25,9 @@ proc readChain*(bitReader: BitReader, symbolDecoder, positionDecoder: HuffmanDec
26 while not symbol.isEndMarker(): 25 while not symbol.isEndMarker():
27 if byteCursor > maxDataByteLength: raise newException(IOError, "lzss block too long") 26 if byteCursor > maxDataByteLength: raise newException(IOError, "lzss block too long")
28 if symbol.isCharacter(): 27 if symbol.isCharacter():
29 chain.append(lzssCharacter(symbol.uint8)) 28 chain.add(lzssCharacter(symbol.uint8))
30 else: 29 else:
31 let position = positionDecoder.decode(bitReader) 30 let position = positionDecoder.decode(bitReader)
32 chain.append(unpackLzssReference(symbol, position)) 31 chain.add(unpackLzssReference(symbol, position))
33 (symbol, byteCursor) = (symbolDecoder.decode(bitReader).Symbol, byteCursor + 1) 32 (symbol, byteCursor) = (symbolDecoder.decode(bitReader).Symbol, byteCursor + 1)
34 chain 33 chain
diff --git a/src/lzsshuffman/lzsshuffmanencoder.nim b/src/lzsshuffman/lzsshuffmanencoder.nim
index ea89f85..205f464 100644
--- a/src/lzsshuffman/lzsshuffmanencoder.nim
+++ b/src/lzsshuffman/lzsshuffmanencoder.nim
@@ -14,9 +14,8 @@
14# You should have received a copy of the GNU Affero General Public License 14# You should have received a copy of the GNU Affero General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>. 15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16 16
17import lists
18import ../bitio/bitwriter 17import ../bitio/bitwriter
19import ../lzss/listpolyfill, ../lzss/lzssnode, ../lzss/lzsschain, ../lzss/lzssencoder 18import ../lzss/lzssnode, ../lzss/lzsschain, ../lzss/lzssencoder
20import ../huffman/huffmantree, ../huffman/huffmantreebuilder, ../huffman/huffmanencoder 19import ../huffman/huffmantree, ../huffman/huffmantreebuilder, ../huffman/huffmanencoder
21