Fix long file names with special characters in containers

commons-lib/src/androidTest/kotlin/ee/ria/DigiDoc/common/ConstantTest.kt

@@ -22,6 +22,7 @@
 package ee.ria.DigiDoc.common
 
 import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
 import org.junit.Assert.assertTrue
 import org.junit.Test
 import java.time.ZoneOffset
@@ -70,8 +71,11 @@ class ConstantTest {
 
     @Test
     fun testFilenameRestrictions() {
-        assertTrue(Constant.RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING.contains('@'))
-        assertTrue(Constant.RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING.contains('\u202E'))
+        assertTrue(Constant.FORBIDDEN_FILENAME_CHARACTERS.contains('/'))
+        assertTrue(Constant.FORBIDDEN_FILENAME_CHARACTERS.contains('*'))
+        assertFalse(Constant.FORBIDDEN_FILENAME_CHARACTERS.contains('@'))
+        // The app adds " (99)", an extension and "-data-files" to a name later
+        assertTrue(Constant.MAX_FILENAME_BYTES + 25 <= 255)
     }
 
     @Test

commons-lib/src/main/kotlin/ee/ria/DigiDoc/common/Constant.kt

@@ -115,10 +115,14 @@ object Constant {
 
     const val TSL_SEQUENCE_NUMBER_ELEMENT: String = "TSLSequenceNumber"
     const val KEY_LOCALE = "locale"
-    private const val RESTRICTED_FILENAME_CHARACTERS_AS_STRING = "@%:^?[]\\'\"”’{}#&`\\\\~«»/´"
-    private const val RTL_CHARACTERS_AS_STRING = "" + '\u200E' + '\u200F' + '\u202E' + '\u202A' + '\u202B'
-    const val RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING =
-        RESTRICTED_FILENAME_CHARACTERS_AS_STRING + RTL_CHARACTERS_AS_STRING
+
+    const val FORBIDDEN_FILENAME_CHARACTERS = "/\\<>:\"|?*"
+
+    const val ZERO_WIDTH_JOINER_CODE = 0x200D
+
+    // 255 bytes is the file system limit. The rest is room for the " (99)",
+    // extension and "-data-files" that the app adds on top of this name.
+    const val MAX_FILENAME_BYTES = 230
     const val DEFAULT_FILENAME = "newFile"
     const val ALLOWED_URL_CHARACTERS =
         "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_,.:/%;+=@?&!()"

crypto-lib/src/main/kotlin/ee/ria/DigiDoc/cryptolib/CryptoContainer.kt

@@ -40,6 +40,7 @@ import ee.ria.DigiDoc.utilsLib.container.ContainerUtil
 import ee.ria.DigiDoc.utilsLib.extensions.isCryptoContainer
 import ee.ria.DigiDoc.utilsLib.extensions.saveAs
 import ee.ria.DigiDoc.utilsLib.file.FileUtil.sanitizeString
+import ee.ria.DigiDoc.utilsLib.file.FileUtil.uniqueFileName
 import ee.ria.DigiDoc.utilsLib.logging.LoggingUtil.Companion.debugLog
 import ee.ria.DigiDoc.utilsLib.logging.LoggingUtil.Companion.errorLog
 import ee.ria.cdoc.CDoc
@@ -167,7 +168,8 @@ class CryptoContainer
 
         companion object {
             val logger = JavaLogger()
-            var loggingIsSet = false
+
+            private val libcdocLogLevel = LogLevel.LEVEL_TRACE
 
             // Encryption can pause and resume midway, so it needs the coroutine-friendly Mutex.
             // Decryption runs start-to-finish in one go, so a plain lock (ReentrantLock) is enough.
@@ -307,15 +309,17 @@ class CryptoContainer
                         debugLog(LOG_TAG, "Decryption started for container ${file.name}")
 
                         val fileInfo = FileInfo()
+                        val savedNames = mutableSetOf<String>()
+                        val dir = ContainerUtil.getContainerDataFilesDir(context, file)
+
+                        // Some file names crash the app if libcdoc logs them while reading
+                        logger.setMinLogLevel(LogLevel.LEVEL_INFO)
+
                         var result: Long = cdocReader.nextFile(fileInfo)
                         while (result == CDoc.OK.toLong()) {
                             val ofile = File(fileInfo.name)
-                            val dir =
-                                ContainerUtil.getContainerDataFilesDir(
-                                    context,
-                                    file,
-                                )
-                            val tmp = sanitizeString(ofile.name, "")
+                            val tmp = uniqueFileName(sanitizeString(ofile.name, ""), savedNames)
+                            savedNames.add(tmp)
                             val fileToSave = File(dir, tmp)
                             FileOutputStream(fileToSave).use { ofs ->
                                 cdocReader.readFile(ofs)
@@ -342,6 +346,7 @@ class CryptoContainer
                         errorLog(LOG_TAG, "IO Exception while decrypting container: ${exc.message}", exc)
                         throw CryptoException("IO Exception: ${exc.message}", exc)
                     } finally {
+                        logger.setMinLogLevel(libcdocLogLevel)
                         cdocReader.delete()
                     }
                 }
@@ -527,11 +532,8 @@ class CryptoContainer
 
             fun setLogging(isLoggingEnabled: Boolean) {
                 if (isLoggingEnabled) {
-                    logger.setMinLogLevel(LogLevel.LEVEL_TRACE)
-                    if (!loggingIsSet) {
-                        CDoc.setLogger(logger)
-                        loggingIsSet = true
-                    }
+                    logger.setMinLogLevel(libcdocLogLevel)
+                    CDoc.setLogger(logger)
                     CDoc.log(LogLevel.LEVEL_DEBUG, "CryptoContainer", 450, "Set libcdoc logging: true")
                 }
             }

utils-lib/src/main/kotlin/ee/ria/DigiDoc/utilsLib/container/ContainerUtil.kt

@@ -207,6 +207,7 @@ object ContainerUtil {
             if (i > 0) {
                 name.append(i)
             }
+
             dir = File(directory, name.toString())
             if (dir.isDirectory || !dir.exists()) {
                 break
@@ -220,6 +221,8 @@ object ContainerUtil {
             if (directory != null) {
                 debugLog(LOG_TAG, "Directories created or already exist for " + directory.path)
             }
+        } else if (!dir.isDirectory) {
+            errorLog(LOG_TAG, "Unable to create data file directory, name is ${dir.name.toByteArray().size} bytes")
         }
 
         return dir

utils-lib/src/main/kotlin/ee/ria/DigiDoc/utilsLib/file/FileUtil.kt

@@ -28,7 +28,9 @@ import android.webkit.URLUtil
 import androidx.core.net.toUri
 import ee.ria.DigiDoc.common.Constant.ALLOWED_URL_CHARACTERS
 import ee.ria.DigiDoc.common.Constant.DEFAULT_FILENAME
-import ee.ria.DigiDoc.common.Constant.RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING
+import ee.ria.DigiDoc.common.Constant.FORBIDDEN_FILENAME_CHARACTERS
+import ee.ria.DigiDoc.common.Constant.MAX_FILENAME_BYTES
+import ee.ria.DigiDoc.common.Constant.ZERO_WIDTH_JOINER_CODE
 import ee.ria.DigiDoc.utilsLib.logging.LoggingUtil.Companion.errorLog
 import ee.ria.DigiDoc.utilsLib.logging.LoggingUtil.Companion.infoLog
 import kotlinx.coroutines.Dispatchers
@@ -55,6 +57,8 @@ import java.io.OutputStreamWriter
 import java.nio.charset.Charset
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
+import java.text.BreakIterator
+import java.text.Normalizer
 import javax.xml.parsers.DocumentBuilderFactory
 import javax.xml.parsers.SAXParserFactory
 
@@ -124,26 +128,101 @@ object FileUtil {
         } else if (trimmed.startsWith(".")) {
             trimmed = DEFAULT_FILENAME + trimmed
         }
+        if (isRawUrl(trimmed)) {
+            return FilenameUtils.getName(FilenameUtils.normalize(trimmed)) ?: DEFAULT_FILENAME
+        }
+        if (URLUtil.isValidUrl(trimmed)) {
+            return FilenameUtils.getName(normalizeUri(trimmed.toUri()).toString()) ?: DEFAULT_FILENAME
+        }
         val sb = StringBuilder(trimmed.length)
-        if (!URLUtil.isValidUrl(trimmed) && !isRawUrl(trimmed)) {
-            for (element in trimmed) {
-                if (RESTRICTED_FILENAME_CHARACTERS_AND_RTL_CHARACTERS_AS_STRING.indexOf(element) != -1) {
-                    sb.append(replacement)
-                } else {
-                    sb.append(element)
-                }
+        for (element in trimmed) {
+            if (isForbiddenInFileName(element)) {
+                sb.append(replacement)
+            } else {
+                sb.append(element)
             }
-        } else if (!isRawUrl(trimmed)) {
-            return normalizeUri(trimmed.toUri()).toString()
         }
-        return if (sb.toString().isNotEmpty()) {
+        val name: String =
             FilenameUtils.getName(
                 FilenameUtils.normalize(
-                    sb.toString(),
+                    sb.toString().trim { it <= ' ' },
                 ),
-            )
+            ) ?: ""
+        return if (name.isEmpty() || name.all { it == '.' }) {
+            DEFAULT_FILENAME
         } else {
-            FilenameUtils.normalize(trimmed)
+            truncateFileName(Normalizer.normalize(name, Normalizer.Form.NFC), MAX_FILENAME_BYTES)
+        }
+    }
+
+    private fun isForbiddenInFileName(character: Char): Boolean {
+        if (character.code == ZERO_WIDTH_JOINER_CODE) {
+            return false
+        }
+        return FORBIDDEN_FILENAME_CHARACTERS.indexOf(character) != -1 ||
+            character.category == CharCategory.CONTROL ||
+            character.category == CharCategory.FORMAT
+    }
+
+    fun truncateFileName(
+        fileName: String,
+        maxBytes: Int,
+    ): String {
+        if (fileName.toByteArray().size <= maxBytes) {
+            return fileName
+        }
+        val extension = FilenameUtils.getExtension(fileName)
+        val suffix = if (extension.isEmpty()) "" else ".$extension"
+        val baseName = FilenameUtils.getBaseName(fileName)
+
+        val shortenedBase = truncateToBytes(baseName, maxBytes - suffix.toByteArray().size)
+        if (shortenedBase.isNotEmpty()) {
+            return shortenedBase + suffix
+        }
+
+        val shortenedName = truncateToBytes(fileName, maxBytes)
+        if (shortenedName.isNotEmpty()) {
+            return shortenedName
+        }
+
+        return truncateToBytes(DEFAULT_FILENAME + suffix, maxBytes)
+    }
+
+    private fun truncateToBytes(
+        text: String,
+        maxBytes: Int,
+    ): String {
+        val characters = BreakIterator.getCharacterInstance()
+        characters.setText(text)
+
+        var lastFit = characters.first()
+        var bytes = 0
+        for (next in generateSequence { characters.next().takeIf { it != BreakIterator.DONE } }) {
+            bytes += text.substring(lastFit, next).toByteArray().size
+            if (bytes > maxBytes) {
+                break
+            }
+            lastFit = next
+        }
+        return text.substring(0, lastFit)
+    }
+
+    fun uniqueFileName(
+        fileName: String,
+        taken: Set<String>,
+    ): String {
+        if (!taken.contains(fileName)) {
+            return fileName
+        }
+        val baseName = FilenameUtils.getBaseName(fileName).ifEmpty { DEFAULT_FILENAME }
+        val extension = FilenameUtils.getExtension(fileName)
+        var counter = 1
+        while (true) {
+            val candidate = if (extension.isEmpty()) "$baseName ($counter)" else "$baseName ($counter).$extension"
+            if (!taken.contains(candidate)) {
+                return candidate
+            }
+            counter++
         }
     }