diff --git a/app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java b/app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java index e081294..43881af 100644 --- a/app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java +++ b/app/src/androidTest/java/com/maxistar/textpad/test/EditorRecoveryTest.java @@ -215,6 +215,50 @@ public void successfulSaveAsRemovesUntitledRecoveryAndUsesNamedIdentity() throws } } + @Test + public void utf16leWithBomRoundTripsThroughRecoveryRestoreAndSave() throws Exception { + Assume.assumeTrue(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q); + String content = "Hello UTF-16LE"; + byte[] utf16leBytes = content.getBytes(java.nio.charset.Charset.forName("UTF-16LE")); + byte[] withBom = new byte[2 + utf16leBytes.length]; + withBom[0] = (byte) 0xFF; + withBom[1] = (byte) 0xFE; + java.lang.System.arraycopy(utf16leBytes, 0, withBom, 2, utf16leBytes.length); + + Uri documentUri = createTestDocumentRaw(withBom); + String key = RecoveryKeys.forDocumentUri(documentUri.toString()); + new RecoveryRepository(context).write( + metadataWithEncoding(key, documentUri.toString(), "UTF-16LE", true), + content + ); + + Intent intent = new Intent(context, EditorActivity.class) + .setAction(Intent.ACTION_VIEW) + .setData(documentUri); + + try (ActivityScenario scenario = ActivityScenario.launch(intent)) { + onView(withText(R.string.Restore)).perform(click()); + onView(withId(R.id.editText1)).check(matches(withText(content))); + scenario.onActivity(activity -> { + EditText editor = activity.findViewById(R.id.editText1); + editor.setSelection(content.length()); + editor.getText().append(" modified"); + }); + android.os.SystemClock.sleep(800); + scenario.recreate(); + onView(withText(R.string.Restore)).perform(click()); + scenario.onActivity(activity -> invokeNoArgument(activity, "saveNamedFile")); + byte[] savedBytes = readDocumentRawBytes(documentUri); + assertEquals(0xFF, savedBytes[0] & 0xFF); + assertEquals(0xFE, savedBytes[1] & 0xFF); + String savedText = new String(savedBytes, 2, savedBytes.length - 2, + java.nio.charset.Charset.forName("UTF-16LE")); + assertEquals("Hello UTF-16LE modified", savedText); + } finally { + context.getContentResolver().delete(documentUri, null, null); + } + } + private void assertLargeDocumentStateIsBinderSafe(boolean simpleScrolling) { setSimpleScrolling(simpleScrolling); String content = generatedDocument(1_050_000); @@ -261,6 +305,15 @@ private RecoveryMetadata metadata(String key, String documentUri) { ); } + private RecoveryMetadata metadataWithEncoding(String key, String documentUri, + String encoding, boolean hasBom) { + return new RecoveryMetadata( + key, documentUri, documentUri == null ? "newfile.txt" : "notes.txt", + documentUri == null, encoding, hasBom, + null, null, null, 0, 0, 0, 0, 1 + ); + } + private Uri createTestDocument(String content) throws Exception { ContentValues values = new ContentValues(); values.put(MediaStore.MediaColumns.DISPLAY_NAME, "textpad-recovery-" + java.lang.System.nanoTime() + ".txt"); @@ -296,6 +349,39 @@ private String readDocument(Uri uri) { } } + private Uri createTestDocumentRaw(byte[] rawContent) throws Exception { + ContentValues values = new ContentValues(); + values.put(MediaStore.MediaColumns.DISPLAY_NAME, "textpad-recovery-" + java.lang.System.nanoTime() + ".txt"); + values.put(MediaStore.MediaColumns.MIME_TYPE, "text/plain"); + values.put(MediaStore.MediaColumns.RELATIVE_PATH, "Download/TextPadTests"); + Uri uri = context.getContentResolver().insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, values); + if (uri == null) { + throw new IllegalStateException("Unable to create test document"); + } + try (java.io.OutputStream output = context.getContentResolver().openOutputStream(uri, "wt")) { + if (output == null) { + throw new IllegalStateException("Unable to write test document"); + } + output.write(rawContent); + } + return uri; + } + + private byte[] readDocumentRawBytes(Uri uri) throws Exception { + try (InputStream input = context.getContentResolver().openInputStream(uri); + ByteArrayOutputStream output = new ByteArrayOutputStream()) { + if (input == null) { + throw new IllegalStateException("Unable to read test document"); + } + byte[] buffer = new byte[1024]; + int count; + while ((count = input.read(buffer)) != -1) { + output.write(buffer, 0, count); + } + return output.toByteArray(); + } + } + private static void invokeNoArgument(EditorActivity activity, String methodName) { try { Method method = EditorActivity.class.getDeclaredMethod(methodName); diff --git a/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java b/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java index dc2795b..cec31f4 100644 --- a/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java +++ b/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java @@ -70,6 +70,7 @@ import com.maxistar.textpad.recovery.RecoveryWriter; import com.maxistar.textpad.utils.EditTextUndoRedo; import com.maxistar.textpad.utils.DocumentSaveValidator; +import com.maxistar.textpad.utils.FileEncoding; import com.maxistar.textpad.utils.FileNameHelper; import com.maxistar.textpad.utils.System; import com.maxistar.textpad.utils.TextConverter; @@ -130,6 +131,8 @@ public class EditorActivity extends AppCompatActivity { private ScrollView scrollView; private LinearLayout linearLayout; + private FileEncoding documentEncoding; + String urlFilename = TPStrings.EMPTY; Uri lastTriedSystemUri = null; @@ -261,6 +264,13 @@ private boolean simpleScrolling() { return settingsService.isUseSimpleScrolling(); } + private String resolveFileEncodingName() { + if (documentEncoding != null) { + return documentEncoding.getCharsetName(); + } + return settingsService.getFileEncoding(); + } + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) private void applyEdgeToEdgeInsets() { View editorRoot = findViewById(R.id.editor_root); @@ -534,6 +544,9 @@ private void restoreDraft(RecoveryDraft draft) { originalSize = draft.metadata.originalSize; originalLastModified = draft.metadata.originalLastModified; originalContentSha256 = draft.metadata.originalContentSha256; + if (draft.metadata.encoding != null && !draft.metadata.encoding.isEmpty()) { + documentEncoding = FileEncoding.fromCharset(draft.metadata.encoding, draft.metadata.hasBom); + } setEditorText(draft.text, true); updateTitle(); if (!draft.metadata.untitled) { @@ -594,8 +607,8 @@ private RecoveryWriter.Snapshot createRecoverySnapshot() { identity, currentDisplayName(), identity == null, - settingsService.getFileEncoding(), - false, + resolveFileEncodingName(), + documentEncoding != null && documentEncoding.hasBom(), originalSize, originalLastModified, originalContentSha256, @@ -1159,6 +1172,7 @@ public void clearFile() { originalSize = null; originalLastModified = null; originalContentSha256 = null; + documentEncoding = null; selectionStart = 0; selectionEnd = 0; setEditorText(TPStrings.EMPTY, false); @@ -1340,7 +1354,7 @@ protected void saveFile(Uri uri) throws IOException { s = applyEndings(s); - outputStream.write(s.getBytes(settingsService.getFileEncoding())); + outputStream.write(FileEncoding.encode(s, documentEncoding, settingsService.getFileEncoding())); } finally { outputStream.close(); } @@ -1356,7 +1370,7 @@ private void guardedSaveNamedFile(boolean autosave) { SaveRequest request = new SaveRequest( editorGeneration, recoveryKey, - persistedText.getBytes(settingsService.getFileEncoding()) + FileEncoding.encode(persistedText, documentEncoding, settingsService.getFileEncoding()) ); boolean creatingDocument = nextSaveCreatesDocument || originalContentSha256 == null; nextSaveCreatesDocument = false; @@ -1551,8 +1565,11 @@ private void validateOpenDocumentOnForeground() { return; } - byte[] intendedBytes = applyEndings(mText.getText().toString()) - .getBytes(settingsService.getFileEncoding()); + byte[] intendedBytes = FileEncoding.encode( + applyEndings(mText.getText().toString()), + documentEncoding, + settingsService.getFileEncoding() + ); SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes); DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify( currentBytes, @@ -1572,7 +1589,8 @@ private void validateOpenDocumentOnForeground() { } private void applyExternalDocument(byte[] externalBytes) throws Exception { - String externalText = new String(externalBytes, settingsService.getFileEncoding()); + documentEncoding = FileEncoding.detect(externalBytes); + String externalText = FileEncoding.decode(externalBytes, documentEncoding, settingsService.getFileEncoding()); externalText = toUnixEndings(externalText); setEditorText(externalText, false); initEditor(); @@ -1588,8 +1606,11 @@ private void validateRestoredDraft() { return; } try { - byte[] intendedBytes = applyEndings(mText.getText().toString()) - .getBytes(settingsService.getFileEncoding()); + byte[] intendedBytes = FileEncoding.encode( + applyEndings(mText.getText().toString()), + documentEncoding, + settingsService.getFileEncoding() + ); SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes); byte[] currentBytes = readNamedDocumentBytes(); DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify( @@ -1664,9 +1685,8 @@ private void openNamedFileLegacyDirect(String filename) { dis.close(); fis.close(); - String ttt = new String(b, 0, length, - settingsService.getFileEncoding()); - + documentEncoding = FileEncoding.detect(b); + String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding()); ttt = toUnixEndings(ttt); setEditorText(ttt, false); @@ -1717,7 +1737,8 @@ private void openNamedFileDirect(final Uri uri) { } byte[] b = bytes.toByteArray(); - String ttt = new String(b, settingsService.getFileEncoding()); + documentEncoding = FileEncoding.detect(b); + String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding()); ttt = toUnixEndings(ttt); inputStream.close(); diff --git a/app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java b/app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java index 7772b46..1b11fcc 100644 --- a/app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java +++ b/app/src/main/java/com/maxistar/textpad/recovery/RecoveryMetadata.java @@ -144,7 +144,7 @@ public JSONObject toJson() throws JSONException { value.put("documentUri", documentUri == null ? JSONObject.NULL : documentUri); value.put("displayName", displayName); value.put("isUntitled", untitled); - value.put("encoding", encoding); + value.put("encoding", encoding == null ? "" : encoding); value.put("hasBom", hasBom); value.put("originalSize", originalSize == null ? JSONObject.NULL : originalSize); value.put("originalLastModified", originalLastModified == null ? JSONObject.NULL : originalLastModified); @@ -169,7 +169,7 @@ public static RecoveryMetadata fromJson(JSONObject value) throws JSONException { nullableString(value, "documentUri"), value.optString("displayName", ""), value.getBoolean("isUntitled"), - value.optString("encoding", "UTF-8"), + nullableStringWithFallback(value, "encoding", "UTF-8"), value.optBoolean("hasBom", false), nullableLong(value, "originalSize"), nullableLong(value, "originalLastModified"), @@ -187,6 +187,14 @@ private static String nullableString(JSONObject value, String name) throws JSONE return value.isNull(name) ? null : value.getString(name); } + private static String nullableStringWithFallback(JSONObject value, String name, String fallback) throws JSONException { + if (value.isNull(name) || !value.has(name)) { + return fallback; + } + String result = value.getString(name); + return result == null || result.isEmpty() ? fallback : result; + } + private static Long nullableLong(JSONObject value, String name) throws JSONException { return value.isNull(name) ? null : value.getLong(name); } diff --git a/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java b/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java new file mode 100644 index 0000000..cdbfc2a --- /dev/null +++ b/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java @@ -0,0 +1,135 @@ +package com.maxistar.textpad.utils; + +import java.nio.charset.Charset; + +/** + * Detects a text file encoding from its byte order mark (BOM) and decodes or + * encodes the content keeping the original encoding. + */ +public class FileEncoding { + + public static final String UTF_8 = "UTF-8"; + public static final String UTF_16LE = "UTF-16LE"; + public static final String UTF_16BE = "UTF-16BE"; + public static final String UTF_32LE = "UTF-32LE"; + public static final String UTF_32BE = "UTF-32BE"; + + private static final byte[] BOM_UTF_32LE = {(byte) 0xFF, (byte) 0xFE, 0, 0}; + private static final byte[] BOM_UTF_32BE = {0, 0, (byte) 0xFE, (byte) 0xFF}; + private static final byte[] BOM_UTF_8 = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}; + private static final byte[] BOM_UTF_16BE = {(byte) 0xFE, (byte) 0xFF}; + private static final byte[] BOM_UTF_16LE = {(byte) 0xFF, (byte) 0xFE}; + + private final String charsetName; + private final byte[] bom; + + private FileEncoding(String charsetName, byte[] bom) { + this.charsetName = charsetName; + this.bom = bom; + } + + public String getCharsetName() { + return charsetName; + } + + public byte[] getBom() { + return bom; + } + + public boolean hasBom() { + return bom != null; + } + + public static FileEncoding detect(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + if (startsWith(bytes, BOM_UTF_32LE)) { + return new FileEncoding(UTF_32LE, BOM_UTF_32LE); + } + if (startsWith(bytes, BOM_UTF_32BE)) { + return new FileEncoding(UTF_32BE, BOM_UTF_32BE); + } + if (startsWith(bytes, BOM_UTF_8)) { + return new FileEncoding(UTF_8, BOM_UTF_8); + } + if (startsWith(bytes, BOM_UTF_16BE)) { + return new FileEncoding(UTF_16BE, BOM_UTF_16BE); + } + if (startsWith(bytes, BOM_UTF_16LE)) { + return new FileEncoding(UTF_16LE, BOM_UTF_16LE); + } + return null; + } + + public static FileEncoding fromCharset(String charsetName, boolean hasBom) { + if (charsetName == null || charsetName.isEmpty()) { + return null; + } + if (!hasBom) { + return new FileEncoding(charsetName, null); + } + byte[] bom = bomForCharset(charsetName); + return new FileEncoding(charsetName, bom); + } + + private static byte[] bomForCharset(String charsetName) { + if (UTF_32LE.equals(charsetName)) return BOM_UTF_32LE; + if (UTF_32BE.equals(charsetName)) return BOM_UTF_32BE; + if (UTF_8.equals(charsetName)) return BOM_UTF_8; + if (UTF_16BE.equals(charsetName)) return BOM_UTF_16BE; + if (UTF_16LE.equals(charsetName)) return BOM_UTF_16LE; + return null; + } + + public static String decode(byte[] bytes, FileEncoding encoding, String fallbackCharsetName) { + if (bytes == null) { + return ""; + } + int offset = 0; + if (encoding != null && encoding.hasBom() && bytes.length >= encoding.getBom().length) { + offset = encoding.getBom().length; + } + String charsetName = encoding != null ? encoding.getCharsetName() : fallbackCharsetName; + try { + return new String(bytes, offset, bytes.length - offset, charsetForName(charsetName)); + } catch (Exception e) { + return new String(bytes); + } + } + + public static byte[] encode(String text, FileEncoding encoding, String fallbackCharsetName) { + String charsetName = encoding != null ? encoding.getCharsetName() : fallbackCharsetName; + byte[] body = text.getBytes(charsetForName(charsetName)); + if (encoding != null && encoding.hasBom()) { + byte[] result = new byte[encoding.getBom().length + body.length]; + java.lang.System.arraycopy(encoding.getBom(), 0, result, 0, encoding.getBom().length); + java.lang.System.arraycopy(body, 0, result, encoding.getBom().length, body.length); + return result; + } + return body; + } + + private static Charset charsetForName(String charsetName) { + if (charsetName == null) { + return Charset.defaultCharset(); + } + try { + return Charset.forName(charsetName); + } catch (Exception e) { + return Charset.defaultCharset(); + } + } + + private static boolean startsWith(byte[] bytes, byte[] prefix) { + if (bytes.length < prefix.length) { + return false; + } + for (int i = 0; i < prefix.length; i++) { + if (bytes[i] != prefix[i]) { + return false; + } + } + return true; + } +} \ No newline at end of file diff --git a/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java b/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java new file mode 100644 index 0000000..10e2850 --- /dev/null +++ b/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java @@ -0,0 +1,94 @@ +package com.maxistar.textpad.utils; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class FileEncodingTest { + + private static byte[] bom(byte[]... parts) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte[] part : parts) { + out.write(part, 0, part.length); + } + return out.toByteArray(); + } + + @Test + public void detectUtf16LeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0x41, 0x00}); + assertEquals(FileEncoding.UTF_16LE, encoding.getCharsetName()); + assertEquals(2, encoding.getBom().length); + } + + @Test + public void detectUtf16BeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFE, (byte) 0xFF, 0x00, 0x41}); + assertEquals(FileEncoding.UTF_16BE, encoding.getCharsetName()); + } + + @Test + public void detectUtf8Bom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF, 0x41}); + assertEquals(FileEncoding.UTF_8, encoding.getCharsetName()); + } + + @Test + public void detectUtf32LeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0, 0, 0x41, 0, 0, 0}); + assertEquals(FileEncoding.UTF_32LE, encoding.getCharsetName()); + } + + @Test + public void detectUtf32BeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{0, 0, (byte) 0xFE, (byte) 0xFF, 0, 0, 0, 0x41}); + assertEquals(FileEncoding.UTF_32BE, encoding.getCharsetName()); + } + + @Test + public void detectNoBom() { + assertNull(FileEncoding.detect("hello".getBytes(StandardCharsets.UTF_8))); + assertNull(FileEncoding.detect(new byte[0])); + assertNull(FileEncoding.detect(null)); + } + + @Test + public void decodeStripsBom() { + byte[] bytes = bom(new byte[]{(byte) 0xFF, (byte) 0xFE}, + "\u041f\u0440\u0438\u0432\u0435\u0442".getBytes(StandardCharsets.UTF_16LE)); + FileEncoding encoding = FileEncoding.detect(bytes); + String text = FileEncoding.decode(bytes, encoding, FileEncoding.UTF_8); + assertEquals("\u041f\u0440\u0438\u0432\u0435\u0442", text); + } + + @Test + public void decodeWithoutBomUsesFallback() { + byte[] bytes = "hello".getBytes(StandardCharsets.UTF_8); + String text = FileEncoding.decode(bytes, null, FileEncoding.UTF_8); + assertEquals("hello", text); + } + + @Test + public void encodeRestoresBom() { + byte[] bytes = bom(new byte[]{(byte) 0xFF, (byte) 0xFE}, + "\u041f\u0440\u0438\u0432\u0435\u0442".getBytes(StandardCharsets.UTF_16LE)); + FileEncoding encoding = FileEncoding.detect(bytes); + byte[] encoded = FileEncoding.encode("\u041f\u0440\u0438\u0432\u0435\u0442", encoding, FileEncoding.UTF_8); + assertArrayEquals(bytes, encoded); + } + + @Test + public void encodeRoundTrip() { + byte[] bytes = bom(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}, + "test".getBytes(StandardCharsets.UTF_8)); + FileEncoding encoding = FileEncoding.detect(bytes); + String text = FileEncoding.decode(bytes, encoding, FileEncoding.UTF_8); + byte[] encoded = FileEncoding.encode(text, encoding, FileEncoding.UTF_8); + assertArrayEquals(bytes, encoded); + } +} \ No newline at end of file