PeterCxy/open-keychain
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases 1 Wiki Activity

Merge pull request #1727 from open-keychain/mime-fallback

Browse source 
try to guess if data is text or not
This commit is contained in:
Vincent 2016-02-23 16:01:13 +01:00
parent c1d268a5ec 03e695c650
commit fd24acbf0e
11 changed files with 526 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/InputDataOperation.java
View file

@ -53,6 +53,7 @@ import org.sufficientlysecure.keychain.provider.ProviderHelper;
import org.sufficientlysecure.keychain.provider.TemporaryFileProvider;
import org.sufficientlysecure.keychain.service.InputDataParcel;
import org.sufficientlysecure.keychain.service.input.CryptoInputParcel;
import org.sufficientlysecure.keychain.util.CharsetVerifier;
/** This operation deals with input data, trying to determine its type as it goes.
@ -67,7 +68,7 @@ import org.sufficientlysecure.keychain.service.input.CryptoInputParcel;
*/
public class InputDataOperation extends BaseOperation<InputDataParcel> {
final private byte[] buf = new byte[256];
private final byte[] buf = new byte[256];
public InputDataOperation(Context context, ProviderHelper providerHelper, Progressable progressable) {
super(context, providerHelper, progressable);
@ -326,21 +327,37 @@ public class InputDataOperation extends BaseOperation<InputDataParcel> {
throw new IOException("Error getting file for writing!");
}
// If this data looks like text, we pipe the incoming data into a charset
// decoder, to see if the data is legal for the assumed charset.
String charset = bd.getCharset();
CharsetVerifier charsetVerifier = new CharsetVerifier(buf, mimeType, charset);
int totalLength = 0;
do {
totalLength += len;
out.write(buf, 0, len);
charsetVerifier.readBytesFromBuffer(0, len);
} while ((len = is.read(buf)) > 0);
log.add(LogType.MSG_DATA_MIME_LENGTH, 3, Long.toString(totalLength));
String charset = bd.getCharset();
// the charset defaults to us-ascii, but we want to default to utf-8
if ("us-ascii".equals(charset)) {
charset = "utf-8";
}
OpenPgpMetadata metadata;
if (charsetVerifier.isDefinitelyBinary()) {
metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength);
} else {
if (charsetVerifier.isCharsetFaulty() && charsetVerifier.isCharsetGuessed()) {
log.add(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN, 3, charsetVerifier.getMaybeFaultyCharset());
} else if (charsetVerifier.isCharsetFaulty()) {
log.add(LogType.MSG_DATA_MIME_CHARSET_FAULTY, 3, charsetVerifier.getCharset());
} else if (charsetVerifier.isCharsetGuessed()) {
log.add(LogType.MSG_DATA_MIME_CHARSET_GUESS, 3, charsetVerifier.getCharset());
} else {
log.add(LogType.MSG_DATA_MIME_CHARSET, 3, charsetVerifier.getCharset());
}
OpenPgpMetadata metadata = new OpenPgpMetadata(mFilename, mimeType, 0L, totalLength, charset);
metadata = new OpenPgpMetadata(mFilename, charsetVerifier.getGuessedMimeType(), 0L, totalLength,
charsetVerifier.getCharset());
}
out.close();
outputUris.add(uri);

4
OpenKeychain/src/main/java/org/sufficientlysecure/keychain/operations/results/OperationResult.java
View file

@ -860,6 +860,10 @@ public abstract class OperationResult implements Parcelable {
MSG_DATA_MIME_FROM_EXTENSION (LogLevel.DEBUG, R.string.msg_data_mime_from_extension),
MSG_DATA_MIME_FILENAME (LogLevel.DEBUG, R.string.msg_data_mime_filename),
MSG_DATA_MIME_LENGTH (LogLevel.DEBUG, R.string.msg_data_mime_length),
MSG_DATA_MIME_CHARSET (LogLevel.DEBUG, R.string.msg_data_mime_charset),
MSG_DATA_MIME_CHARSET_FAULTY (LogLevel.WARN, R.string.msg_data_mime_charset_faulty),
MSG_DATA_MIME_CHARSET_GUESS (LogLevel.DEBUG, R.string.msg_data_mime_charset_guess),
MSG_DATA_MIME_CHARSET_UNKNOWN (LogLevel.DEBUG, R.string.msg_data_mime_charset_unknown),
MSG_DATA_MIME (LogLevel.DEBUG, R.string.msg_data_mime),
MSG_DATA_MIME_OK (LogLevel.INFO, R.string.msg_data_mime_ok),
MSG_DATA_MIME_NONE (LogLevel.DEBUG, R.string.msg_data_mime_none),

36
OpenKeychain/src/main/java/org/sufficientlysecure/keychain/pgp/PgpDecryptVerifyOperation.java
View file

@ -35,8 +35,6 @@ import android.support.annotation.NonNull;
import android.text.TextUtils;
import android.webkit.MimeTypeMap;
import org.openintents.openpgp.OpenPgpDecryptionResult;
import org.openintents.openpgp.OpenPgpMetadata;
import org.bouncycastle.bcpg.ArmoredInputStream;
import org.bouncycastle.openpgp.PGPCompressedData;
import org.bouncycastle.openpgp.PGPDataValidationException;
@ -56,10 +54,13 @@ import org.bouncycastle.openpgp.operator.jcajce.CachingDataDecryptorFactory;
import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder;
import org.bouncycastle.openpgp.operator.jcajce.JcePBEDataDecryptorFactoryBuilder;
import org.bouncycastle.util.encoders.DecoderException;
import org.openintents.openpgp.OpenPgpDecryptionResult;
import org.openintents.openpgp.OpenPgpMetadata;
import org.sufficientlysecure.keychain.Constants;
import org.sufficientlysecure.keychain.Constants.key;
import org.sufficientlysecure.keychain.R;
import org.sufficientlysecure.keychain.operations.BaseOperation;
import org.sufficientlysecure.keychain.util.CharsetVerifier;
import org.sufficientlysecure.keychain.operations.results.DecryptVerifyResult;
import org.sufficientlysecure.keychain.operations.results.OperationResult.LogType;
import org.sufficientlysecure.keychain.operations.results.OperationResult.OperationLog;
@ -387,9 +388,9 @@ public class PgpDecryptVerifyOperation extends BaseOperation<PgpDecryptVerifyInp
MimeTypeMap mime = MimeTypeMap.getSingleton();
mimeType = mime.getMimeTypeFromExtension(extension);
}
if (mimeType == null) {
mimeType = "application/octet-stream";
}
}
if (mimeType == null) {
mimeType = "application/octet-stream";
}
if (!"".equals(originalFilename)) {
@ -414,11 +415,9 @@ public class PgpDecryptVerifyOperation extends BaseOperation<PgpDecryptVerifyInp
}
metadata = new OpenPgpMetadata(
originalFilename,
mimeType,
originalFilename, mimeType,
literalData.getModificationTime().getTime(),
originalSize == null ? 0 : originalSize,
charset);
originalSize == null ? 0 : originalSize, charset);
log.add(LogType.MSG_DC_OK_META_ONLY, indent);
DecryptVerifyResult result =
@ -439,6 +438,8 @@ public class PgpDecryptVerifyOperation extends BaseOperation<PgpDecryptVerifyInp
int length;
byte[] buffer = new byte[8192];
byte[] firstBytes = new byte[48];
CharsetVerifier charsetVerifier = new CharsetVerifier(buffer, mimeType, charset);
while ((length = dataIn.read(buffer)) > 0) {
// Log.d(Constants.TAG, "read bytes: " + length);
if (out != null) {
@ -448,6 +449,8 @@ public class PgpDecryptVerifyOperation extends BaseOperation<PgpDecryptVerifyInp
// update signature buffer if signature is also present
signatureChecker.updateSignatureData(buffer, 0, length);
charsetVerifier.readBytesFromBuffer(0, length);
// note down first couple of bytes for "magic bytes" file type detection
if (alreadyWritten == 0) {
System.arraycopy(buffer, 0, firstBytes, 0, length > firstBytes.length ? firstBytes.length : length);
@ -480,18 +483,21 @@ public class PgpDecryptVerifyOperation extends BaseOperation<PgpDecryptVerifyInp
Log.d(Constants.TAG, "decrypt time taken: " + String.format("%.2f", opTime / 1000.0) + "s");
// special treatment to detect pgp mime types
// TODO move into CharsetVerifier? seems like that would be a plausible place for this logic
if (matchesPrefix(firstBytes, "-----BEGIN PGP PUBLIC KEY BLOCK-----")
|| matchesPrefix(firstBytes, "-----BEGIN PGP PRIVATE KEY BLOCK-----")) {
mimeType = Constants.MIME_TYPE_KEYS;
} else if (matchesPrefix(firstBytes, "-----BEGIN PGP MESSAGE-----")) {
// this is NOT application/pgp-encrypted, see RFC 3156!
mimeType = Constants.MIME_TYPE_ENCRYPTED_ALTERNATE;
} else {
mimeType = charsetVerifier.getGuessedMimeType();
}
metadata = new OpenPgpMetadata(originalFilename, mimeType, literalData.getModificationTime().getTime(),
alreadyWritten, charsetVerifier.getCharset());
log.add(LogType.MSG_DC_CLEAR_META_MIME, indent + 1, mimeType);
metadata = new OpenPgpMetadata(
originalFilename, mimeType, literalData.getModificationTime().getTime(), alreadyWritten, charset);
Log.d(Constants.TAG, metadata.toString());
indent -= 1;
@ -873,11 +879,7 @@ public class PgpDecryptVerifyOperation extends BaseOperation<PgpDecryptVerifyInp
log.add(LogType.MSG_DC_OK, indent);
OpenPgpMetadata metadata = new OpenPgpMetadata(
"",
"text/plain",
-1,
clearText.length);
OpenPgpMetadata metadata = new OpenPgpMetadata("", "text/plain", -1, clearText.length, "utf-8");
DecryptVerifyResult result = new DecryptVerifyResult(DecryptVerifyResult.RESULT_OK, log);
result.setSignatureResult(signatureChecker.getSignatureResult());

11
OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptFragment.java
View file

@ -19,6 +19,7 @@ package org.sufficientlysecure.keychain.ui;
import java.util.ArrayList;
import android.app.Activity;
import android.content.Intent;
import android.database.Cursor;
import android.net.Uri;
@ -472,6 +473,16 @@ public abstract class DecryptFragment extends Fragment implements LoaderManager.
protected abstract void onVerifyLoaded(boolean hideErrorOverlay);
public void startDisplayLogActivity() {
Activity activity = getActivity();
if (activity == null) {
return;
}
Intent intent = new Intent(activity, LogDisplayActivity.class);
intent.putExtra(LogDisplayFragment.EXTRA_RESULT, mDecryptVerifyResult);
activity.startActivity(intent);
}
@Override
public void onActivityResult(int requestCode, int resultCode, Intent data) {
if (mImportOpHelper != null) {

12
OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DecryptListFragment.java
View file

@ -591,6 +591,18 @@ public class DecryptListFragment
Intent chooserIntent = Intent.createChooser(intent, getString(R.string.intent_show));
chooserIntent.setFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION);
if (!share && ClipDescription.compareMimeTypes(metadata.getMimeType(), "text/*")) {
LabeledIntent internalIntent = new LabeledIntent(
new Intent(intent)
.setClass(activity, DisplayTextActivity.class)
.putExtra(DisplayTextActivity.EXTRA_RESULT, result.mDecryptVerifyResult)
.putExtra(DisplayTextActivity.EXTRA_METADATA, metadata),
BuildConfig.APPLICATION_ID, R.string.view_internal, R.mipmap.ic_launcher);
chooserIntent.putExtra(Intent.EXTRA_INITIAL_INTENTS,
new Parcelable[] { internalIntent });
}
startActivity(chooserIntent);
}

6
OpenKeychain/src/main/java/org/sufficientlysecure/keychain/ui/DisplayTextFragment.java
View file

@ -103,7 +103,7 @@ public class DisplayTextFragment extends DecryptFragment {
Bundle args = getArguments();
String plaintext = args.getString(ARG_PLAINTEXT);
DecryptVerifyResult result = args.getParcelable(ARG_DECRYPT_VERIFY_RESULT);
DecryptVerifyResult result = args.getParcelable(ARG_DECRYPT_VERIFY_RESULT);
// display signature result in activity
mText.setText(plaintext);
@ -137,6 +137,10 @@ public class DisplayTextFragment extends DecryptFragment {
copyToClipboard(mText.getText().toString());
break;
}
case R.id.decrypt_view_log: {
startDisplayLogActivity();
break;
}
default: {
return super.onOptionsItemSelected(item);
}

154
OpenKeychain/src/main/java/org/sufficientlysecure/keychain/util/CharsetVerifier.java Normal file
View file

@ -0,0 +1,154 @@
package org.sufficientlysecure.keychain.util;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;
import android.content.ClipDescription;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
/** This class can be used to guess whether a stream of data is encoded in a given
* charset or not.
*
* An object of this class must be initialized with a byte[] buffer, which should
* be filled with data, then processed with {@link #readBytesFromBuffer}. This can
* be done any number of times. Once all data has been read, a final status can be
* read using the getter methods.
*/
public class CharsetVerifier {
private final ByteBuffer bufWrap;
private final CharBuffer dummyOutput;
private final CharsetDecoder charsetDecoder;
private boolean isFinished;
private boolean isFaulty;
private boolean isGuessed;
private boolean isPossibleTextMimeType;
private boolean isTextMimeType;
private String charset;
private String mimeType;
public CharsetVerifier(@NonNull byte[] buf, @NonNull String mimeType, @Nullable String charset) {
this.mimeType = mimeType;
isTextMimeType = ClipDescription.compareMimeTypes(mimeType, "text/*");
isPossibleTextMimeType = isTextMimeType
|| ClipDescription.compareMimeTypes(mimeType, "application/octet-stream")
|| ClipDescription.compareMimeTypes(mimeType, "application/x-download");
if (!isPossibleTextMimeType) {
charsetDecoder = null;
bufWrap = null;
dummyOutput = null;
return;
}
bufWrap = ByteBuffer.wrap(buf);
dummyOutput = CharBuffer.allocate(buf.length);
// the charset defaults to us-ascii, but we want to default to utf-8
if (charset == null || "us-ascii".equals(charset)) {
charset = "utf-8";
isGuessed = true;
} else {
isGuessed = false;
}
this.charset = charset;
charsetDecoder = Charset.forName(charset).newDecoder();
charsetDecoder.onMalformedInput(CodingErrorAction.REPORT);
charsetDecoder.onUnmappableCharacter(CodingErrorAction.REPORT);
charsetDecoder.reset();
}
public void readBytesFromBuffer(int pos, int len) {
if (isFinished) {
throw new IllegalStateException("cannot write again after reading charset status!");
}
if (isFaulty || bufWrap == null) {
return;
}
bufWrap.rewind();
bufWrap.position(pos);
bufWrap.limit(len);
dummyOutput.rewind();
CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, false);
if (result.isError()) {
isFaulty = true;
}
}
private void finishIfNecessary() {
if (isFinished || isFaulty || bufWrap == null) {
return;
}
isFinished = true;
bufWrap.rewind();
bufWrap.limit(0);
dummyOutput.rewind();
CoderResult result = charsetDecoder.decode(bufWrap, dummyOutput, true);
if (result.isError()) {
isFaulty = true;
}
}
public String getGuessedMimeType() {
if (isTextMimeType) {
return mimeType;
}
if (isProbablyText()) {
return "text/plain";
}
return mimeType;
}
public boolean isCharsetFaulty() {
finishIfNecessary();
return isFaulty;
}
public boolean isCharsetGuessed() {
finishIfNecessary();
return isGuessed;
}
public String getCharset() {
finishIfNecessary();
if (!isPossibleTextMimeType || (isGuessed && isFaulty)) {
return null;
}
return charset;
}
public String getMaybeFaultyCharset() {
return charset;
}
/** Returns true if the data which was read is definitely binary.
*
* This can happen when either the supplied mimeType indicated a non-ambiguous
* binary data type, or if we guessed a charset but got errors while decoding.
*/
public boolean isDefinitelyBinary() {
finishIfNecessary();
return !isTextMimeType && (!isPossibleTextMimeType || (isGuessed && isFaulty));
}
/** Returns true iff the data which was read is probably (or
* definitely) text.
*
* The corner case where isDefinitelyBinary returns false but isProbablyText
* returns true is where the charset was provided by the data (so is not
* guessed) but is still faulty.
*/
public boolean isProbablyText() {
finishIfNecessary();
return isTextMimeType || isPossibleTextMimeType && (!isGuessed || !isFaulty);
}
}

17
OpenKeychain/src/main/res/menu/decrypt_menu.xml
View file

@ -2,16 +2,21 @@
<menu xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto">
<item
android:id="@+id/decrypt_copy"
android:title="@string/btn_copy_decrypted_text"
android:icon="@drawable/ic_content_copy_black_24dp"
app:showAsAction="ifRoom" />
<item
android:id="@+id/decrypt_share"
android:title="@string/btn_share_decrypted_text"
android:icon="@drawable/ic_share_black_24dp"
app:showAsAction="ifRoom" />
<item
android:id="@+id/decrypt_copy"
android:title="@string/btn_copy_decrypted_text"
android:icon="@drawable/ic_content_copy_black_24dp"
app:showAsAction="never" />
<item
android:id="@+id/decrypt_view_log"
android:title="@string/btn_view_log"
app:showAsAction="never" />
</menu>

6
OpenKeychain/src/main/res/values/strings.xml
View file

@ -95,7 +95,7 @@
<string name="btn_add_files">"Add file(s)"</string>
<string name="btn_share_decrypted_text">"Share"</string>
<string name="btn_open_with">"Open with…"</string>
<string name="btn_copy_decrypted_text">"Copy decrypted text"</string>
<string name="btn_copy_decrypted_text">"Copy to clipboard"</string>
<string name="btn_decrypt_clipboard">"Read from clipboard"</string>
<string name="btn_decrypt_files">"Select input file"</string>
<string name="btn_encrypt_files">"Encrypt files"</string>
@ -1401,6 +1401,10 @@
<string name="msg_data_mime_filename">"Filename: '%s'"</string>
<string name="msg_data_mime_from_extension">"Guessing MIME type from extension"</string>
<string name="msg_data_mime_length">"Content-Length: %s"</string>
<string name="msg_data_mime_charset">"Charset is '%s'"</string>
<string name="msg_data_mime_charset_faulty">"Charset is '%s', but decoding failed!"</string>
<string name="msg_data_mime_charset_guess">"Charset appears to be '%s'"</string>
<string name="msg_data_mime_charset_unknown">"Charset is unknown, or data is not text."</string>
<string name="msg_data_mime">"Parsing MIME data structure"</string>
<string name="msg_data_mime_ok">"Finished parsing"</string>
<string name="msg_data_mime_none">"No MIME structure found"</string>

141
OpenKeychain/src/test/java/org/sufficientlysecure/keychain/pgp/InputDataOperationTest.java
View file

@ -20,6 +20,7 @@ package org.sufficientlysecure.keychain.pgp;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.security.Security;
import java.util.ArrayList;
@ -30,18 +31,20 @@ import android.content.ContentValues;
import android.net.Uri;
import junit.framework.Assert;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.openintents.openpgp.OpenPgpMetadata;
import org.robolectric.RobolectricGradleTestRunner;
import org.robolectric.RuntimeEnvironment;
import org.robolectric.annotation.Config;
import org.robolectric.shadows.ShadowLog;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.sufficientlysecure.keychain.WorkaroundBuildConfig;
import org.sufficientlysecure.keychain.operations.InputDataOperation;
import org.sufficientlysecure.keychain.operations.results.InputDataResult;
import org.sufficientlysecure.keychain.operations.results.OperationResult.LogType;
import org.sufficientlysecure.keychain.provider.ProviderHelper;
import org.sufficientlysecure.keychain.provider.TemporaryFileProvider;
import org.sufficientlysecure.keychain.service.InputDataParcel;
@ -80,20 +83,20 @@ public class InputDataOperationTest {
}
@Test
public void testMimeDecoding () throws Exception {
public void testMimeDecoding() throws Exception {
String mimeMail =
"Content-Type: multipart/mixed; boundary=\"=-26BafqxfXmhVNMbYdoIi\"\n" +
"\n" +
"--=-26BafqxfXmhVNMbYdoIi\n" +
"Content-Type: text/plain\n" +
"Content-Type: text/plain; charset=utf-8\n" +
"Content-Transfer-Encoding: quoted-printable\n" +
"Content-Disposition: attachment; filename=data.txt\n" +
"\n" +
"message part 1\n" +
"\n" +
"--=-26BafqxfXmhVNMbYdoIi\n" +
"Content-Type: text/testvalue\n" +
"Content-Type: text/testvalue; charset=iso-8859-1\n" +
"Content-Description: Dummy content description\n" +
"\n" +
"message part 2.1\n" +
@ -156,7 +159,137 @@ public class InputDataOperationTest {
Assert.assertEquals("second part must have expected content",
"message part 2.1\nmessage part 2.2\n", new String(outStream2.toByteArray()));
OpenPgpMetadata metadata = result.mMetadata.get(0);
Assert.assertEquals("text/plain", metadata.getMimeType());
Assert.assertEquals("utf-8", metadata.getCharset());
metadata = result.mMetadata.get(1);
Assert.assertEquals("text/testvalue", metadata.getMimeType());
Assert.assertEquals("iso-8859-1", metadata.getCharset());
}
@Test
public void testMimeDecodingExplicitFaultyCharset() throws Exception {
String mimeContent = "Content-Type: text/plain; charset=utf-8\n" +
"\n" +
"message with binary data in it\n";
byte[] data = mimeContent.getBytes();
data[60] = (byte) 0xc3;
data[61] = (byte) 0x28;
InputDataResult result = runSimpleDataInputOperation(data);
// must be successful, no verification, have two output URIs
Assert.assertTrue(result.success());
Assert.assertNull(result.mDecryptVerifyResult);
OpenPgpMetadata metadata = result.mMetadata.get(0);
Assert.assertEquals("text/plain", metadata.getMimeType());
Assert.assertEquals("charset should be set since it was explicitly specified",
"utf-8", metadata.getCharset());
Assert.assertTrue("faulty charset should have been detected",
result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_FAULTY));
}
@Test
public void testMimeDecodingImplicitFaultyCharset() throws Exception {
String mimeContent = "Content-Type: text/plain\n" +
"\n" +
"message with binary data in it\n";
byte[] data = mimeContent.getBytes();
data[45] = (byte) 0xc3;
data[46] = (byte) 0x28;
InputDataResult result = runSimpleDataInputOperation(data);
// must be successful, no verification, have two output URIs
Assert.assertTrue(result.success());
Assert.assertNull(result.mDecryptVerifyResult);
OpenPgpMetadata metadata = result.mMetadata.get(0);
Assert.assertEquals("text/plain", metadata.getMimeType());
Assert.assertNull("charset was bad so it should not be set", metadata.getCharset());
Assert.assertTrue("faulty charset should have been detected",
result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_UNKNOWN));
}
@Test
public void testMimeDecodingImplicitGuessedCharset() throws Exception {
String mimeContent = "Content-Type: text/plain\n" +
"\n" +
"proper, utf-8 encoded message ☭\n";
InputDataResult result = runSimpleDataInputOperation(mimeContent.getBytes());
// must be successful, no verification, have two output URIs
Assert.assertTrue(result.success());
Assert.assertNull(result.mDecryptVerifyResult);
OpenPgpMetadata metadata = result.mMetadata.get(0);
Assert.assertEquals("text/plain", metadata.getMimeType());
Assert.assertEquals("charset should be set since it was guessed and not faulty",
"utf-8", metadata.getCharset());
Assert.assertTrue("charset should have been guessed",
result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_GUESS));
}
@Test
public void testMimeDecodingOctetStreamGuessedCharset() throws Exception {
String mimeContent = "Content-Type: application/octet-stream\n" +
"\n" +
"proper, utf-8 encoded message ☭\n";
InputDataResult result = runSimpleDataInputOperation(mimeContent.getBytes());
// must be successful, no verification, have two output URIs
Assert.assertTrue(result.success());
Assert.assertNull(result.mDecryptVerifyResult);
OpenPgpMetadata metadata = result.mMetadata.get(0);
Assert.assertEquals("text/plain", metadata.getMimeType());
Assert.assertEquals("charset should be set since it was guessed and not faulty",
"utf-8", metadata.getCharset());
Assert.assertTrue("charset should have been guessed",
result.getLog().containsType(LogType.MSG_DATA_MIME_CHARSET_GUESS));
}
private InputDataResult runSimpleDataInputOperation(byte[] mimeContentBytes) throws FileNotFoundException {
ByteArrayOutputStream outStream1 = new ByteArrayOutputStream();
ByteArrayOutputStream outStream2 = new ByteArrayOutputStream();
ContentResolver mockResolver = mock(ContentResolver.class);
// fake openOutputStream first and second
when(mockResolver.openOutputStream(any(Uri.class), eq("w")))
.thenReturn(outStream1, outStream2);
// fake openInputStream
Uri fakeInputUri = Uri.parse("content://fake/1");
when(mockResolver.openInputStream(fakeInputUri)).thenReturn(
new ByteArrayInputStream(mimeContentBytes));
Uri fakeOutputUri1 = Uri.parse("content://fake/out/1");
when(mockResolver.insert(eq(TemporaryFileProvider.CONTENT_URI), any(ContentValues.class)))
.thenReturn(fakeOutputUri1);
// application which returns mockresolver
Application spyApplication = spy(RuntimeEnvironment.application);
when(spyApplication.getContentResolver()).thenReturn(mockResolver);
InputDataOperation op = new InputDataOperation(spyApplication,
new ProviderHelper(RuntimeEnvironment.application), null);
InputDataParcel input = new InputDataParcel(fakeInputUri, null);
return op.execute(input, new CryptoInputParcel());
}
}

144
OpenKeychain/src/test/java/org/sufficientlysecure/keychain/util/CharsetVerifierTest.java Normal file
View file

@ -0,0 +1,144 @@
package org.sufficientlysecure.keychain.util;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.RobolectricGradleTestRunner;
import org.robolectric.annotation.Config;
import org.sufficientlysecure.keychain.WorkaroundBuildConfig;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@RunWith(RobolectricGradleTestRunner.class)
@Config(constants = WorkaroundBuildConfig.class, sdk = 21, manifest = "src/main/AndroidManifest.xml")
public class CharsetVerifierTest {
@Test
public void testTypeImagePngAlwaysBinary() throws Exception {
byte[] bytes = "bla bluh ☭".getBytes("utf-8");
CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "image/png", null);
charsetVerifier.readBytesFromBuffer(0, bytes.length);
assertTrue("image/png should be marked as definitely binary", charsetVerifier.isDefinitelyBinary());
assertFalse("image/png should never be marked as, even if it is", charsetVerifier.isProbablyText());
assertNull("charset should be null", charsetVerifier.getCharset());
}
@Test
public void testUtf8SpecifiedButFaulty() throws Exception {
byte[] bytes = "bla bluh ☭".getBytes("utf-8");
bytes[4] = (byte) 0xc3;
bytes[5] = (byte) 0x28;
CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/something", "utf-8");
charsetVerifier.readBytesFromBuffer(0, bytes.length);
assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary());
assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText());
assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty());
assertFalse("charset was specified and should not be marked as guessed", charsetVerifier.isCharsetGuessed());
assertEquals("mimetype should be preserved", "text/something", charsetVerifier.getGuessedMimeType());
assertEquals("charset should be utf-8 since it was given explicitly", "utf-8", charsetVerifier.getCharset());
assertEquals("charset should be utf-8 since it was given explicitly", "utf-8", charsetVerifier.getMaybeFaultyCharset());
}
@Test
public void testUtf8GuessedAndFaulty() throws Exception {
byte[] bytes = "bla bluh ☭".getBytes("utf-8");
bytes[4] = (byte) 0xc3;
bytes[5] = (byte) 0x28;
CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null);
charsetVerifier.readBytesFromBuffer(0, bytes.length);
assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary());
assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText());
assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty());
assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed());
assertNull("charset should be null since the guess was faulty", charsetVerifier.getCharset());
assertEquals("mimetype should be set to text", "text/plain", charsetVerifier.getGuessedMimeType());
assertEquals("maybe-faulty charset should be utf-8", "utf-8", charsetVerifier.getMaybeFaultyCharset());
}
@Test
public void testGuessedEncoding() throws Exception {
byte[] bytes = "bla bluh ☭".getBytes("utf-8");
CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "application/octet-stream", null);
charsetVerifier.readBytesFromBuffer(0, bytes.length);
assertFalse("application/octet-stream with text content is not definitely binary", charsetVerifier.isDefinitelyBinary());
assertTrue("application/octet-stream with text content should be probably text", charsetVerifier.isProbablyText());
assertFalse("detected charset should not be faulty", charsetVerifier.isCharsetFaulty());
assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed());
assertEquals("mimetype should be set to text", "text/plain", charsetVerifier.getGuessedMimeType());
assertEquals("guessed charset is utf-8", "utf-8", charsetVerifier.getCharset());
}
@Test
public void testWindows1252Faulty() throws Exception {
byte[] bytes = "bla bluh ☭".getBytes("windows-1252");
bytes[2] = (byte) 0x9d;
CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "windows-1252");
charsetVerifier.readBytesFromBuffer(0, bytes.length);
assertFalse("text/plain is never definitely binary", charsetVerifier.isDefinitelyBinary());
assertTrue("text/plain is always probably text", charsetVerifier.isProbablyText());
assertTrue("charset contained faulty characters", charsetVerifier.isCharsetFaulty());
assertFalse("charset was not guessed", charsetVerifier.isCharsetGuessed());
assertEquals("charset is returned correctly", "windows-1252", charsetVerifier.getCharset());
}
@Test
public void testWindows1252Good() throws Exception {
byte[] bytes = "bla bluh ☭".getBytes("windows-1252");
// this is ‡ in windows-1252
bytes[2] = (byte) 0x87;
CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", "windows-1252");
charsetVerifier.readBytesFromBuffer(0, bytes.length);
assertFalse("text/plain is never definitely binary", charsetVerifier.isDefinitelyBinary());
assertTrue("text/plain is always probably text", charsetVerifier.isProbablyText());
assertFalse("charset contained no faulty characters", charsetVerifier.isCharsetFaulty());
assertFalse("charset was not guessed", charsetVerifier.isCharsetGuessed());
assertEquals("charset is returned correctly", "windows-1252", charsetVerifier.getCharset());
}
@Test(expected = IllegalStateException.class)
public void testReadAfterGetterShouldCrash() throws Exception {
byte[] bytes = "bla bluh ☭".getBytes("utf-8");
CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null);
charsetVerifier.readBytesFromBuffer(0, bytes.length);
charsetVerifier.isCharsetFaulty();
charsetVerifier.readBytesFromBuffer(0, bytes.length);
}
@Test
public void testStaggeredInput() throws Exception {
byte[] bytes = "bla bluh ☭".getBytes("utf-8");
bytes[4] = (byte) 0xc3;
bytes[5] = (byte) 0x28;
CharsetVerifier charsetVerifier = new CharsetVerifier(bytes, "text/plain", null);
for (int i = 0; i < bytes.length; i++) {
charsetVerifier.readBytesFromBuffer(i, i+1);
}
assertFalse("text/plain should not be marked as binary, even if it is", charsetVerifier.isDefinitelyBinary());
assertTrue("text/plain should be marked as text, even if it isn't valid", charsetVerifier.isProbablyText());
assertTrue("encoding contained illegal chars, so it should be marked as faulty", charsetVerifier.isCharsetFaulty());
assertTrue("charset was guessed and should be marked as such", charsetVerifier.isCharsetGuessed());
assertNull("charset should be null since the guess was faulty", charsetVerifier.getCharset());
assertEquals("maybe-faulty charset should be utf-8", "utf-8", charsetVerifier.getMaybeFaultyCharset());
}
}