From 3075833ab3812f7dbce8886fd750950fb376dd48 Mon Sep 17 00:00:00 2001 From: Daniel Gultsch Date: Wed, 15 Sep 2021 11:38:03 +0200 Subject: [PATCH] swap out transcoder library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the transcoder library we used hasn’t been updated in years this commit switches to a maintained fork https://natario1.github.io/Transcoder/ --- build.gradle | 3 +- .../AttachFileToConversationRunnable.java | 300 +++++++++--------- .../utils/Android360pFormatStrategy.java | 76 ----- .../utils/Android720pFormatStrategy.java | 76 ----- .../utils/TranscoderStrategies.java | 41 +++ 5 files changed, 196 insertions(+), 300 deletions(-) delete mode 100644 src/main/java/eu/siacs/conversations/utils/Android360pFormatStrategy.java delete mode 100644 src/main/java/eu/siacs/conversations/utils/Android720pFormatStrategy.java create mode 100644 src/main/java/eu/siacs/conversations/utils/TranscoderStrategies.java diff --git a/build.gradle b/build.gradle index 1fcf2d8bc..7bf12f01a 100644 --- a/build.gradle +++ b/build.gradle @@ -64,7 +64,8 @@ dependencies { implementation 'org.whispersystems:signal-protocol-java:2.6.2' implementation 'com.makeramen:roundedimageview:2.3.0' implementation "com.wefika:flowlayout:0.4.1" - implementation 'net.ypresto.androidtranscoder:android-transcoder:0.3.0' + implementation 'com.otaliastudios:transcoder:0.10.3' + implementation 'org.jxmpp:jxmpp-jid:1.0.1' implementation 'org.osmdroid:osmdroid-android:6.1.10' implementation 'org.hsluv:hsluv:0.2' diff --git a/src/main/java/eu/siacs/conversations/services/AttachFileToConversationRunnable.java b/src/main/java/eu/siacs/conversations/services/AttachFileToConversationRunnable.java index 5d39911ed..db879799d 100644 --- a/src/main/java/eu/siacs/conversations/services/AttachFileToConversationRunnable.java +++ b/src/main/java/eu/siacs/conversations/services/AttachFileToConversationRunnable.java @@ -3,16 +3,19 @@ package eu.siacs.conversations.services; import android.content.Context; import android.content.SharedPreferences; import android.net.Uri; -import android.os.ParcelFileDescriptor; import android.preference.PreferenceManager; import android.util.Log; -import net.ypresto.androidtranscoder.MediaTranscoder; -import net.ypresto.androidtranscoder.format.MediaFormatStrategy; +import androidx.annotation.NonNull; + +import com.otaliastudios.transcoder.Transcoder; +import com.otaliastudios.transcoder.TranscoderListener; + +import org.jetbrains.annotations.NotNull; import java.io.File; -import java.io.FileDescriptor; import java.io.FileNotFoundException; +import java.util.Objects; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -23,161 +26,164 @@ import eu.siacs.conversations.entities.DownloadableFile; import eu.siacs.conversations.entities.Message; import eu.siacs.conversations.persistance.FileBackend; import eu.siacs.conversations.ui.UiCallback; -import eu.siacs.conversations.utils.Android360pFormatStrategy; -import eu.siacs.conversations.utils.Android720pFormatStrategy; import eu.siacs.conversations.utils.MimeUtils; +import eu.siacs.conversations.utils.TranscoderStrategies; -public class AttachFileToConversationRunnable implements Runnable, MediaTranscoder.Listener { +public class AttachFileToConversationRunnable implements Runnable, TranscoderListener { - private final XmppConnectionService mXmppConnectionService; - private final Message message; - private final Uri uri; - private final String type; - private final UiCallback callback; - private final boolean isVideoMessage; - private final long originalFileSize; - private int currentProgress = -1; + private final XmppConnectionService mXmppConnectionService; + private final Message message; + private final Uri uri; + private final String type; + private final UiCallback callback; + private final boolean isVideoMessage; + private final long originalFileSize; + private int currentProgress = -1; - AttachFileToConversationRunnable(XmppConnectionService xmppConnectionService, Uri uri, String type, Message message, UiCallback callback) { - this.uri = uri; - this.type = type; - this.mXmppConnectionService = xmppConnectionService; - this.message = message; - this.callback = callback; - final String mimeType = MimeUtils.guessMimeTypeFromUriAndMime(mXmppConnectionService, uri, type); - final int autoAcceptFileSize = mXmppConnectionService.getResources().getInteger(R.integer.auto_accept_filesize); - this.originalFileSize = FileBackend.getFileSize(mXmppConnectionService,uri); - this.isVideoMessage = (mimeType != null && mimeType.startsWith("video/")) && originalFileSize > autoAcceptFileSize && !"uncompressed".equals(getVideoCompression()); - } + AttachFileToConversationRunnable(XmppConnectionService xmppConnectionService, Uri uri, String type, Message message, UiCallback callback) { + this.uri = uri; + this.type = type; + this.mXmppConnectionService = xmppConnectionService; + this.message = message; + this.callback = callback; + final String mimeType = MimeUtils.guessMimeTypeFromUriAndMime(mXmppConnectionService, uri, type); + final int autoAcceptFileSize = mXmppConnectionService.getResources().getInteger(R.integer.auto_accept_filesize); + this.originalFileSize = FileBackend.getFileSize(mXmppConnectionService, uri); + this.isVideoMessage = (mimeType != null && mimeType.startsWith("video/")) && originalFileSize > autoAcceptFileSize && !"uncompressed".equals(getVideoCompression()); + } - boolean isVideoMessage() { - return this.isVideoMessage; - } + boolean isVideoMessage() { + return this.isVideoMessage; + } - private void processAsFile() { - final String path = mXmppConnectionService.getFileBackend().getOriginalPath(uri); - if (path != null && !FileBackend.isPathBlacklisted(path)) { - message.setRelativeFilePath(path); - mXmppConnectionService.getFileBackend().updateFileParams(message); - if (message.getEncryption() == Message.ENCRYPTION_DECRYPTED) { - mXmppConnectionService.getPgpEngine().encrypt(message, callback); - } else { - mXmppConnectionService.sendMessage(message); - callback.success(message); - } - } else { - try { - mXmppConnectionService.getFileBackend().copyFileToPrivateStorage(message, uri, type); - mXmppConnectionService.getFileBackend().updateFileParams(message); - if (message.getEncryption() == Message.ENCRYPTION_DECRYPTED) { - final PgpEngine pgpEngine = mXmppConnectionService.getPgpEngine(); - if (pgpEngine != null) { - pgpEngine.encrypt(message, callback); - } else if (callback != null) { - callback.error(R.string.unable_to_connect_to_keychain, null); - } - } else { - mXmppConnectionService.sendMessage(message); - callback.success(message); - } - } catch (FileBackend.FileCopyException e) { - callback.error(e.getResId(), message); - } - } - } + private void processAsFile() { + final String path = mXmppConnectionService.getFileBackend().getOriginalPath(uri); + if (path != null && !FileBackend.isPathBlacklisted(path)) { + message.setRelativeFilePath(path); + mXmppConnectionService.getFileBackend().updateFileParams(message); + if (message.getEncryption() == Message.ENCRYPTION_DECRYPTED) { + mXmppConnectionService.getPgpEngine().encrypt(message, callback); + } else { + mXmppConnectionService.sendMessage(message); + callback.success(message); + } + } else { + try { + mXmppConnectionService.getFileBackend().copyFileToPrivateStorage(message, uri, type); + mXmppConnectionService.getFileBackend().updateFileParams(message); + if (message.getEncryption() == Message.ENCRYPTION_DECRYPTED) { + final PgpEngine pgpEngine = mXmppConnectionService.getPgpEngine(); + if (pgpEngine != null) { + pgpEngine.encrypt(message, callback); + } else if (callback != null) { + callback.error(R.string.unable_to_connect_to_keychain, null); + } + } else { + mXmppConnectionService.sendMessage(message); + callback.success(message); + } + } catch (FileBackend.FileCopyException e) { + callback.error(e.getResId(), message); + } + } + } - private void processAsVideo() throws FileNotFoundException { - Log.d(Config.LOGTAG,"processing file as video"); - mXmppConnectionService.startForcingForegroundNotification(); - message.setRelativeFilePath(message.getUuid() + ".mp4"); - final DownloadableFile file = mXmppConnectionService.getFileBackend().getFile(message); - final MediaFormatStrategy formatStrategy = "720".equals(getVideoCompression()) ? new Android720pFormatStrategy() : new Android360pFormatStrategy(); - file.getParentFile().mkdirs(); - final ParcelFileDescriptor parcelFileDescriptor = mXmppConnectionService.getContentResolver().openFileDescriptor(uri, "r"); - if (parcelFileDescriptor == null) { - throw new FileNotFoundException("Parcel File Descriptor was null"); - } - FileDescriptor fileDescriptor = parcelFileDescriptor.getFileDescriptor(); - Future future = MediaTranscoder.getInstance().transcodeVideo(fileDescriptor, file.getAbsolutePath(), formatStrategy, this); - try { - future.get(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } catch (ExecutionException e) { - if (e.getCause() instanceof Error) { - mXmppConnectionService.stopForcingForegroundNotification(); - processAsFile(); - } else { - Log.d(Config.LOGTAG, "ignoring execution exception. Should get handled by onTranscodeFiled() instead", e); - } - } - } + private void processAsVideo() throws FileNotFoundException { + Log.d(Config.LOGTAG, "processing file as video"); + mXmppConnectionService.startForcingForegroundNotification(); + message.setRelativeFilePath(message.getUuid() + ".mp4"); + final DownloadableFile file = mXmppConnectionService.getFileBackend().getFile(message); + if (Objects.requireNonNull(file.getParentFile()).mkdirs()) { + Log.d(Config.LOGTAG, "created parent directory for video file"); + } - @Override - public void onTranscodeProgress(double progress) { - final int p = (int) Math.round(progress * 100); - if (p > currentProgress) { - currentProgress = p; - mXmppConnectionService.getNotificationService().updateFileAddingNotification(p,message); - } - } + final boolean highQuality = "720".equals(getVideoCompression()); - @Override - public void onTranscodeCompleted() { - mXmppConnectionService.stopForcingForegroundNotification(); - final File file = mXmppConnectionService.getFileBackend().getFile(message); - long convertedFileSize = mXmppConnectionService.getFileBackend().getFile(message).getSize(); - Log.d(Config.LOGTAG,"originalFileSize="+originalFileSize+" convertedFileSize="+convertedFileSize); - if (originalFileSize != 0 && convertedFileSize >= originalFileSize) { - if (file.delete()) { - Log.d(Config.LOGTAG,"original file size was smaller. deleting and processing as file"); - processAsFile(); - return; - } else { - Log.d(Config.LOGTAG,"unable to delete converted file"); - } - } - mXmppConnectionService.getFileBackend().updateFileParams(message); - if (message.getEncryption() == Message.ENCRYPTION_DECRYPTED) { - mXmppConnectionService.getPgpEngine().encrypt(message, callback); - } else { - mXmppConnectionService.sendMessage(message); - callback.success(message); - } - } + final Future future = Transcoder.into(file.getAbsolutePath()). + addDataSource(mXmppConnectionService, uri) + .setVideoTrackStrategy(highQuality ? TranscoderStrategies.VIDEO_720P : TranscoderStrategies.VIDEO_360P) + .setAudioTrackStrategy(highQuality ? TranscoderStrategies.AUDIO_HQ : TranscoderStrategies.AUDIO_MQ) + .setListener(this) + .transcode(); + try { + future.get(); + } catch (InterruptedException e) { + throw new AssertionError(e); + } catch (ExecutionException e) { + if (e.getCause() instanceof Error) { + mXmppConnectionService.stopForcingForegroundNotification(); + processAsFile(); + } else { + Log.d(Config.LOGTAG, "ignoring execution exception. Should get handled by onTranscodeFiled() instead", e); + } + } + } - @Override - public void onTranscodeCanceled() { - mXmppConnectionService.stopForcingForegroundNotification(); - processAsFile(); - } + @Override + public void onTranscodeProgress(double progress) { + final int p = (int) Math.round(progress * 100); + if (p > currentProgress) { + currentProgress = p; + mXmppConnectionService.getNotificationService().updateFileAddingNotification(p, message); + } + } - @Override - public void onTranscodeFailed(Exception e) { - mXmppConnectionService.stopForcingForegroundNotification(); - Log.d(Config.LOGTAG,"video transcoding failed",e); - processAsFile(); - } + @Override + public void onTranscodeCompleted(int successCode) { + mXmppConnectionService.stopForcingForegroundNotification(); + final File file = mXmppConnectionService.getFileBackend().getFile(message); + long convertedFileSize = mXmppConnectionService.getFileBackend().getFile(message).getSize(); + Log.d(Config.LOGTAG, "originalFileSize=" + originalFileSize + " convertedFileSize=" + convertedFileSize); + if (originalFileSize != 0 && convertedFileSize >= originalFileSize) { + if (file.delete()) { + Log.d(Config.LOGTAG, "original file size was smaller. deleting and processing as file"); + processAsFile(); + return; + } else { + Log.d(Config.LOGTAG, "unable to delete converted file"); + } + } + mXmppConnectionService.getFileBackend().updateFileParams(message); + if (message.getEncryption() == Message.ENCRYPTION_DECRYPTED) { + mXmppConnectionService.getPgpEngine().encrypt(message, callback); + } else { + mXmppConnectionService.sendMessage(message); + callback.success(message); + } + } - @Override - public void run() { - if (this.isVideoMessage()) { - try { - processAsVideo(); - } catch (FileNotFoundException e) { - processAsFile(); - } - } else { - processAsFile(); - } - } + @Override + public void onTranscodeCanceled() { + mXmppConnectionService.stopForcingForegroundNotification(); + processAsFile(); + } - private String getVideoCompression() { - return getVideoCompression(mXmppConnectionService); - } + @Override + public void onTranscodeFailed(@NonNull @NotNull Throwable exception) { + mXmppConnectionService.stopForcingForegroundNotification(); + Log.d(Config.LOGTAG, "video transcoding failed", exception); + processAsFile(); + } - public static String getVideoCompression(final Context context) { - final SharedPreferences preferences = PreferenceManager.getDefaultSharedPreferences(context); - return preferences.getString("video_compression", context.getResources().getString(R.string.video_compression)); - } + @Override + public void run() { + if (this.isVideoMessage()) { + try { + processAsVideo(); + } catch (FileNotFoundException e) { + processAsFile(); + } + } else { + processAsFile(); + } + } + + private String getVideoCompression() { + return getVideoCompression(mXmppConnectionService); + } + + public static String getVideoCompression(final Context context) { + final SharedPreferences preferences = PreferenceManager.getDefaultSharedPreferences(context); + return preferences.getString("video_compression", context.getResources().getString(R.string.video_compression)); + } } diff --git a/src/main/java/eu/siacs/conversations/utils/Android360pFormatStrategy.java b/src/main/java/eu/siacs/conversations/utils/Android360pFormatStrategy.java deleted file mode 100644 index a692cc6dc..000000000 --- a/src/main/java/eu/siacs/conversations/utils/Android360pFormatStrategy.java +++ /dev/null @@ -1,76 +0,0 @@ -package eu.siacs.conversations.utils; - -import android.media.MediaCodecInfo; -import android.media.MediaFormat; -import android.os.Build; -import android.util.Log; - -import androidx.annotation.RequiresApi; - -import net.ypresto.androidtranscoder.format.MediaFormatExtraConstants; -import net.ypresto.androidtranscoder.format.MediaFormatStrategy; -import net.ypresto.androidtranscoder.format.OutputFormatUnavailableException; - -import eu.siacs.conversations.Config; - -public class Android360pFormatStrategy implements MediaFormatStrategy { - - private static final int LONGER_LENGTH = 640; - private static final int SHORTER_LENGTH = 360; - private static final int DEFAULT_VIDEO_BITRATE = 1000 * 1000; - private static final int DEFAULT_AUDIO_BITRATE = 128 * 1000; - private final int mVideoBitrate; - private final int mAudioBitrate; - private final int mAudioChannels; - - public Android360pFormatStrategy() { - mVideoBitrate = DEFAULT_VIDEO_BITRATE; - mAudioBitrate = DEFAULT_AUDIO_BITRATE; - mAudioChannels = 2; - } - - @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) - @Override - public MediaFormat createVideoOutputFormat(MediaFormat inputFormat) { - int width = inputFormat.getInteger(MediaFormat.KEY_WIDTH); - int height = inputFormat.getInteger(MediaFormat.KEY_HEIGHT); - int longer, shorter, outWidth, outHeight; - if (width >= height) { - longer = width; - shorter = height; - outWidth = LONGER_LENGTH; - outHeight = SHORTER_LENGTH; - } else { - shorter = width; - longer = height; - outWidth = SHORTER_LENGTH; - outHeight = LONGER_LENGTH; - } - if (longer * 9 != shorter * 16) { - throw new OutputFormatUnavailableException("This video is not 16:9, and is not able to transcode. (" + width + "x" + height + ")"); - } - if (shorter <= SHORTER_LENGTH) { - Log.d(Config.LOGTAG, "This video is less or equal to 360p, pass-through. (" + width + "x" + height + ")"); - return null; - } - MediaFormat format = MediaFormat.createVideoFormat("video/avc", outWidth, outHeight); - format.setInteger(MediaFormat.KEY_BIT_RATE, mVideoBitrate); - format.setInteger(MediaFormat.KEY_FRAME_RATE, 30); - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 3); - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - format.setInteger(MediaFormat.KEY_PROFILE ,MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline); - format.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel13); - } - return format; - } - - @Override - public MediaFormat createAudioOutputFormat(MediaFormat inputFormat) { - final MediaFormat format = MediaFormat.createAudioFormat(MediaFormatExtraConstants.MIMETYPE_AUDIO_AAC, inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE), mAudioChannels); - format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - format.setInteger(MediaFormat.KEY_BIT_RATE, mAudioBitrate); - return format; - } - -} diff --git a/src/main/java/eu/siacs/conversations/utils/Android720pFormatStrategy.java b/src/main/java/eu/siacs/conversations/utils/Android720pFormatStrategy.java deleted file mode 100644 index 274ebb76f..000000000 --- a/src/main/java/eu/siacs/conversations/utils/Android720pFormatStrategy.java +++ /dev/null @@ -1,76 +0,0 @@ -package eu.siacs.conversations.utils; - -import android.media.MediaCodecInfo; -import android.media.MediaFormat; -import android.os.Build; -import android.util.Log; - -import androidx.annotation.RequiresApi; - -import net.ypresto.androidtranscoder.format.MediaFormatExtraConstants; -import net.ypresto.androidtranscoder.format.MediaFormatStrategy; -import net.ypresto.androidtranscoder.format.OutputFormatUnavailableException; - -import eu.siacs.conversations.Config; - -public class Android720pFormatStrategy implements MediaFormatStrategy { - - private static final int LONGER_LENGTH = 1280; - private static final int SHORTER_LENGTH = 720; - private static final int DEFAULT_VIDEO_BITRATE = 2000 * 1000; - private static final int DEFAULT_AUDIO_BITRATE = 192 * 1000; - private final int mVideoBitrate; - private final int mAudioBitrate; - private final int mAudioChannels; - - public Android720pFormatStrategy() { - mVideoBitrate = DEFAULT_VIDEO_BITRATE; - mAudioBitrate = DEFAULT_AUDIO_BITRATE; - mAudioChannels = 2; - } - - @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) - @Override - public MediaFormat createVideoOutputFormat(MediaFormat inputFormat) { - int width = inputFormat.getInteger(MediaFormat.KEY_WIDTH); - int height = inputFormat.getInteger(MediaFormat.KEY_HEIGHT); - int longer, shorter, outWidth, outHeight; - if (width >= height) { - longer = width; - shorter = height; - outWidth = LONGER_LENGTH; - outHeight = SHORTER_LENGTH; - } else { - shorter = width; - longer = height; - outWidth = SHORTER_LENGTH; - outHeight = LONGER_LENGTH; - } - if (longer * 9 != shorter * 16) { - throw new OutputFormatUnavailableException("This video is not 16:9, and is not able to transcode. (" + width + "x" + height + ")"); - } - if (shorter <= SHORTER_LENGTH) { - Log.d(Config.LOGTAG, "This video is less or equal to 720p, pass-through. (" + width + "x" + height + ")"); - return null; - } - MediaFormat format = MediaFormat.createVideoFormat("video/avc", outWidth, outHeight); - format.setInteger(MediaFormat.KEY_BIT_RATE, mVideoBitrate); - format.setInteger(MediaFormat.KEY_FRAME_RATE, 30); - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 3); - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - format.setInteger(MediaFormat.KEY_PROFILE ,MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline); - format.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel13); - } - return format; - } - - @Override - public MediaFormat createAudioOutputFormat(MediaFormat inputFormat) { - final MediaFormat format = MediaFormat.createAudioFormat(MediaFormatExtraConstants.MIMETYPE_AUDIO_AAC, inputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE), mAudioChannels); - format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - format.setInteger(MediaFormat.KEY_BIT_RATE, mAudioBitrate); - return format; - } - -} diff --git a/src/main/java/eu/siacs/conversations/utils/TranscoderStrategies.java b/src/main/java/eu/siacs/conversations/utils/TranscoderStrategies.java new file mode 100644 index 000000000..0fb0766d1 --- /dev/null +++ b/src/main/java/eu/siacs/conversations/utils/TranscoderStrategies.java @@ -0,0 +1,41 @@ +package eu.siacs.conversations.utils; + +import com.otaliastudios.transcoder.strategy.DefaultAudioStrategy; +import com.otaliastudios.transcoder.strategy.DefaultVideoStrategy; + +public final class TranscoderStrategies { + + public static final DefaultVideoStrategy VIDEO_720P = DefaultVideoStrategy.atMost(720) + .bitRate(2L * 1000 * 1000) + .frameRate(30) + .keyFrameInterval(3F) + .build(); + + public static final DefaultVideoStrategy VIDEO_360P = DefaultVideoStrategy.atMost(360) + .bitRate(1000 * 1000) + .frameRate(30) + .keyFrameInterval(3F) + .build(); + + //TODO do we want to add 240p (@500kbs) and 1080p (@4mbs?) ? + // see suggested bit rates on https://www.videoproc.com/media-converter/bitrate-setting-for-h264.htm + + public static final DefaultAudioStrategy AUDIO_HQ = DefaultAudioStrategy.builder() + .bitRate(192 * 1000) + .channels(2) + .sampleRate(DefaultAudioStrategy.SAMPLE_RATE_AS_INPUT) + .build(); + + public static final DefaultAudioStrategy AUDIO_MQ = DefaultAudioStrategy.builder() + .bitRate(128 * 1000) + .channels(2) + .sampleRate(DefaultAudioStrategy.SAMPLE_RATE_AS_INPUT) + .build(); + + //TODO if we add 144p we definitely want to add a lower audio bit rate as well + + private TranscoderStrategies() { + throw new IllegalStateException("Do not instantiate me"); + } + +}