Calibre: More QoL tweaks (#7545)

* Wireless: Optimize memory usage in StreamMessageQueue (use an array of string ropes, that we only concatenate once). Allowed to relax the throttling, making transfers that much faster.
* Persist: Add a "zstd" codec, that uses the "luajit" codec, but compressed via zstd. Since both of those are very fast, it pretty much trounces everything in terms of speed and size ;).
* Persist: Implemented a "writes_to_file" framework, much like the existing "reads_from_file" one. And use it in the zstd codec to avoid useless temporary string interning.
* Metadata: Switch to the zstd codec.

frontend/ui/message/streammessagequeue.lua

@@ -56,13 +56,18 @@ function StreamMessageQueue:handleZframe(frame)
 end
 
 function StreamMessageQueue:waitEvent()
-    local data = ""
     -- Successive zframes may come in batches of tens or hundreds in some cases.
-    -- If they are concatenated in a single loop, it may consume a significant amount
-    -- of memory. And it's fairly easy to trigger when receiving file data from Calibre.
-    -- So, throttle reception to 10 packages at most in one waitEvent loop,
+    -- Since we buffer each frame's data in a Lua string,
+    -- and then let the caller concatenate those,
+    -- it may consume a significant amount of memory.
+    -- And it's fairly easy to trigger when receiving file data from Calibre.
+    -- So, throttle reception to 256 packages at most in one waitEvent loop,
     -- after which we immediately call receiveCallback.
-    local wait_packages = 10
+    local wait_packages = 256
+    -- In a similar spirit, much like LuaSocket,
+    -- we store the data as ropes of strings in an array,
+    -- to be concatenated by the caller.
+    local t = {}
     while czmq.zpoller_wait(self.poller, 0) ~= nil and wait_packages > 0 do
         local id_frame = czmq.zframe_recv(self.socket)
         if id_frame ~= nil then
@@ -70,12 +75,15 @@ function StreamMessageQueue:waitEvent()
         end
         local frame = czmq.zframe_recv(self.socket)
         if frame ~= nil then
-            data = data .. (self:handleZframe(frame) or "")
+            local data = self:handleZframe(frame)
+            if data then
+                table.insert(t, data)
+            end
         end
         wait_packages = wait_packages - 1
     end
-    if self.receiveCallback and data ~= "" then
-        self.receiveCallback(data)
+    if self.receiveCallback and #t ~= 0 then
+        self.receiveCallback(t)
     end
 end