diff --git a/src/main/java/eu/siacs/conversations/generator/MessageGenerator.java b/src/main/java/eu/siacs/conversations/generator/MessageGenerator.java index 41de4ec30..a0cb0ca1f 100644 --- a/src/main/java/eu/siacs/conversations/generator/MessageGenerator.java +++ b/src/main/java/eu/siacs/conversations/generator/MessageGenerator.java @@ -20,6 +20,7 @@ import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xmpp.chatstate.ChatState; import eu.siacs.conversations.xmpp.jingle.JingleConnectionManager; import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection; +import eu.siacs.conversations.xmpp.jingle.Media; import eu.siacs.conversations.xmpp.stanzas.MessagePacket; import rocks.xmpp.addr.Jid; @@ -241,7 +242,10 @@ public class MessageGenerator extends AbstractGenerator { packet.setId(JingleRtpConnection.JINGLE_MESSAGE_PROPOSE_ID_PREFIX + proposal.sessionId); final Element propose = packet.addChild("propose", Namespace.JINGLE_MESSAGE); propose.setAttribute("id", proposal.sessionId); - propose.addChild("description", Namespace.JINGLE_APPS_RTP); + for (final Media media : proposal.media) { + propose.addChild("description", Namespace.JINGLE_APPS_RTP).setAttribute("media", media.toString()); + } + packet.addChild("request", "urn:xmpp:receipts"); packet.addChild("store", "urn:xmpp:hints"); return packet; diff --git a/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java b/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java index a696d1c39..72db2d88e 100644 --- a/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java +++ b/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java @@ -17,6 +17,7 @@ import android.widget.Toast; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.webrtc.RendererCommon; import org.webrtc.SurfaceViewRenderer; @@ -36,6 +37,7 @@ import eu.siacs.conversations.services.XmppConnectionService; import eu.siacs.conversations.utils.PermissionUtils; import eu.siacs.conversations.xmpp.jingle.AbstractJingleConnection; import eu.siacs.conversations.xmpp.jingle.JingleRtpConnection; +import eu.siacs.conversations.xmpp.jingle.Media; import eu.siacs.conversations.xmpp.jingle.RtpEndUserState; import rocks.xmpp.addr.Jid; @@ -199,7 +201,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe resetIntent(intent.getExtras()); } } else if (asList(ACTION_MAKE_VIDEO_CALL, ACTION_MAKE_VOICE_CALL).contains(intent.getAction())) { - proposeJingleRtpSession(account, with); + proposeJingleRtpSession(account, with, ImmutableSet.of(Media.AUDIO)); binding.with.setText(account.getRoster().getContact(with).getDisplayName()); } else if (Intent.ACTION_VIEW.equals(intent.getAction())) { final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE); @@ -213,8 +215,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } } - private void proposeJingleRtpSession(final Account account, final Jid with) { - xmppConnectionService.getJingleConnectionManager().proposeJingleRtpSession(account, with); + private void proposeJingleRtpSession(final Account account, final Jid with, final Set media) { + xmppConnectionService.getJingleConnectionManager().proposeJingleRtpSession(account, with, media); //TODO maybe we don’t want to acquire a wake lock just yet and wait for audio manager to discover what speaker we are using putScreenInCallMode(); } @@ -482,7 +484,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe final Account account = extractAccount(intent); final Jid with = Jid.of(intent.getStringExtra(EXTRA_WITH)); this.rtpConnectionReference = null; - proposeJingleRtpSession(account, with); + proposeJingleRtpSession(account, with, ImmutableSet.of(Media.AUDIO)); } private void exit(View view) { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java index e6f17ac7f..e56105c2e 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java @@ -3,13 +3,21 @@ package eu.siacs.conversations.xmpp.jingle; import android.util.Base64; import android.util.Log; +import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; + +import org.checkerframework.checker.nullness.compatqual.NullableDecl; import java.lang.ref.WeakReference; import java.security.SecureRandom; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import eu.siacs.conversations.Config; @@ -25,8 +33,11 @@ import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xmpp.OnIqPacketReceived; import eu.siacs.conversations.xmpp.jingle.stanzas.Content; import eu.siacs.conversations.xmpp.jingle.stanzas.FileTransferDescription; +import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription; import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; +import eu.siacs.conversations.xmpp.jingle.stanzas.Propose; import eu.siacs.conversations.xmpp.jingle.stanzas.Reason; +import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; import eu.siacs.conversations.xmpp.stanzas.IqPacket; import eu.siacs.conversations.xmpp.stanzas.MessagePacket; import rocks.xmpp.addr.Jid; @@ -129,9 +140,9 @@ public class JingleConnectionManager extends AbstractConnectionManager { } return; } - final boolean addressedToSelf = from.asBareJid().equals(account.getJid().asBareJid()); + final boolean fromSelf = from.asBareJid().equals(account.getJid().asBareJid()); final AbstractJingleConnection.Id id; - if (addressedToSelf) { + if (fromSelf) { if (to.isFullJid()) { id = AbstractJingleConnection.Id.of(account, to, sessionId); } else { @@ -150,15 +161,26 @@ public class JingleConnectionManager extends AbstractConnectionManager { return; } - if (addressedToSelf) { + if (fromSelf) { Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": ignore jingle message from self"); + return; } if ("propose".equals(message.getName())) { - final Element description = message.findChild("description"); - final String namespace = description == null ? null : description.getNamespace(); - if (Namespace.JINGLE_APPS_RTP.equals(namespace) && !usesTor(account)) { - if (isBusy()) { + final Propose propose = Propose.upgrade(message); + final List descriptions = propose.getDescriptions(); + final Collection rtpDescriptions = Collections2.transform( + Collections2.filter(descriptions, d -> d instanceof RtpDescription), + input -> (RtpDescription) input + ); + if (rtpDescriptions.size() > 0 && rtpDescriptions.size() == descriptions.size() && !usesTor(account)) { + final Collection media = Collections2.transform(rtpDescriptions, RtpDescription::getMedia); + if (media.contains(Media.UNKNOWN)) { + Log.d(Config.LOGTAG,account.getJid().asBareJid()+": encountered unknown media in session proposal. "+propose); + return; + } + if (isBusy()) { //TODO only if no other devices are active + //TODO create final MessagePacket reject = mXmppConnectionService.getMessageGenerator().sessionReject(from, sessionId); mXmppConnectionService.sendMessagePacket(account, reject); } else { @@ -167,14 +189,15 @@ public class JingleConnectionManager extends AbstractConnectionManager { rtpConnection.deliveryMessage(from, message, serverMsgId, timestamp); } } else { - Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": unable to react to proposed " + namespace + " session"); + Log.d(Config.LOGTAG, account.getJid().asBareJid() + ": unable to react to proposed session with " + rtpDescriptions.size() + " rtp descriptions of " + descriptions.size() + " total descriptions"); } } else if ("proceed".equals(message.getName())) { - - final RtpSessionProposal proposal = new RtpSessionProposal(account, from.asBareJid(), sessionId); synchronized (rtpSessionProposals) { - if (rtpSessionProposals.remove(proposal) != null) { + final RtpSessionProposal proposal = getRtpSessionProposal(account,from.asBareJid(),sessionId); + if (proposal != null) { + rtpSessionProposals.remove(proposal); final JingleRtpConnection rtpConnection = new JingleRtpConnection(this, id, account.getJid()); + rtpConnection.setProposedMedia(proposal.media); this.connections.put(id, rtpConnection); rtpConnection.transitionOrThrow(AbstractJingleConnection.State.PROPOSED); rtpConnection.deliveryMessage(from, message, serverMsgId, timestamp); @@ -198,6 +221,15 @@ public class JingleConnectionManager extends AbstractConnectionManager { } + private RtpSessionProposal getRtpSessionProposal(final Account account, Jid from, String sessionId) { + for(RtpSessionProposal rtpSessionProposal : rtpSessionProposals.keySet()) { + if (rtpSessionProposal.sessionId.equals(sessionId) && rtpSessionProposal.with.equals(from) && rtpSessionProposal.account.getJid().equals(account.getJid())) { + return rtpSessionProposal; + } + } + return null; + } + private void writeLogMissedOutgoing(final Account account, Jid with, final String sessionId, String serverMsgId, long timestamp) { final Conversation conversation = mXmppConnectionService.findOrCreateConversation( account, @@ -310,7 +342,7 @@ public class JingleConnectionManager extends AbstractConnectionManager { } } - public void proposeJingleRtpSession(final Account account, final Jid with) { + public void proposeJingleRtpSession(final Account account, final Jid with, final Set media) { synchronized (this.rtpSessionProposals) { for (Map.Entry entry : this.rtpSessionProposals.entrySet()) { RtpSessionProposal proposal = entry.getKey(); @@ -327,7 +359,7 @@ public class JingleConnectionManager extends AbstractConnectionManager { } } } - final RtpSessionProposal proposal = RtpSessionProposal.of(account, with.asBareJid()); + final RtpSessionProposal proposal = RtpSessionProposal.of(account, with.asBareJid(), media); this.rtpSessionProposals.put(proposal, DeviceDiscoveryState.SEARCHING); mXmppConnectionService.notifyJingleRtpConnectionUpdate( account, @@ -456,15 +488,21 @@ public class JingleConnectionManager extends AbstractConnectionManager { public final Jid with; public final String sessionId; private final Account account; + public final Set media; private RtpSessionProposal(Account account, Jid with, String sessionId) { + this(account,with,sessionId, Collections.emptySet()); + } + + private RtpSessionProposal(Account account, Jid with, String sessionId, Set media) { this.account = account; this.with = with; this.sessionId = sessionId; + this.media = media; } - public static RtpSessionProposal of(Account account, Jid with) { - return new RtpSessionProposal(account, with, nextRandomId()); + public static RtpSessionProposal of(Account account, Jid with, Set media) { + return new RtpSessionProposal(account, with, nextRandomId(), media); } @Override diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index b7284cc4a..7daabe163 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -4,9 +4,12 @@ import android.os.SystemClock; import android.util.Log; import com.google.common.base.Optional; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import org.webrtc.EglBase; @@ -30,10 +33,13 @@ import eu.siacs.conversations.entities.RtpSessionStatus; import eu.siacs.conversations.services.AppRTCAudioManager; import eu.siacs.conversations.xml.Element; import eu.siacs.conversations.xml.Namespace; +import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription; import eu.siacs.conversations.xmpp.jingle.stanzas.Group; import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket; +import eu.siacs.conversations.xmpp.jingle.stanzas.Propose; import eu.siacs.conversations.xmpp.jingle.stanzas.Reason; +import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; import eu.siacs.conversations.xmpp.stanzas.IqPacket; import eu.siacs.conversations.xmpp.stanzas.MessagePacket; import rocks.xmpp.addr.Jid; @@ -108,6 +114,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web private final ArrayDeque pendingIceCandidates = new ArrayDeque<>(); private final Message message; private State state = State.NULL; + private Set proposedMedia; private RtpContentMap initiatorRtpContentMap; private RtpContentMap responderRtpContentMap; private long rtpConnectionStarted = 0; //time of 'connected' @@ -406,7 +413,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message); switch (message.getName()) { case "propose": - receivePropose(from, serverMessageId, timestamp); + receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp); break; case "proceed": receiveProceed(from, serverMessageId, timestamp); @@ -475,11 +482,19 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } } - private void receivePropose(final Jid from, final String serverMsgId, final long timestamp) { + private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) { final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid()); if (originatedFromMyself) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from mysql. ignoring"); } else if (transition(State.PROPOSED)) { + final Collection descriptions = Collections2.transform( + Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription), + input -> (RtpDescription) input + ); + final Collection media = Collections2.transform(descriptions, RtpDescription::getMedia); + Preconditions.checkState(!media.contains(Media.UNKNOWN),"RTP descriptions contain unknown media"); + Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": received session proposal from "+from+" for "+media); + this.proposedMedia = Sets.newHashSet(media); if (serverMsgId != null) { this.message.setServerMsgId(serverMsgId); } @@ -1002,6 +1017,10 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web return webRTCWrapper.getEglBaseContext(); } + public void setProposedMedia(final Set media) { + + } + private interface OnIceServersDiscovered { void onIceServersDiscovered(List iceServers); } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java new file mode 100644 index 000000000..da25516ca --- /dev/null +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java @@ -0,0 +1,20 @@ +package eu.siacs.conversations.xmpp.jingle; + +import java.util.Locale; + +public enum Media { + VIDEO, AUDIO, UNKNOWN; + + @Override + public String toString() { + return super.toString().toLowerCase(Locale.ROOT); + } + + public static Media of(String value) { + try { + return value == null ? UNKNOWN : Media.valueOf(value.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + return UNKNOWN; + } + } +} diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java index 96339061a..db9902874 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java @@ -5,13 +5,16 @@ import android.util.Log; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.checkerframework.checker.nullness.compatqual.NullableDecl; import java.util.Map; +import java.util.Set; import eu.siacs.conversations.Config; import eu.siacs.conversations.xmpp.jingle.stanzas.Content; @@ -48,6 +51,13 @@ public class RtpContentMap { return new RtpContentMap(group, contentMapBuilder.build()); } + public Set getMedia() { + return Sets.newHashSet(Collections2.transform(contents.values(), input -> { + final RtpDescription rtpDescription = input == null ? null : input.description; + return rtpDescription == null ? Media.UNKNOWN : input.description.getMedia(); + })); + } + public void requireContentDescriptions() { if (this.contents.size() == 0) { throw new IllegalStateException("No contents available"); diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Propose.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Propose.java new file mode 100644 index 000000000..da3a93da3 --- /dev/null +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/Propose.java @@ -0,0 +1,41 @@ +package eu.siacs.conversations.xmpp.jingle.stanzas; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import eu.siacs.conversations.xml.Element; +import eu.siacs.conversations.xml.Namespace; + +public class Propose extends Element { + private Propose() { + super("propose", Namespace.JINGLE_MESSAGE); + } + + public List getDescriptions() { + final ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (final Element child : this.children) { + if ("description".equals(child.getName())) { + final String namespace = child.getNamespace(); + if (FileTransferDescription.NAMESPACES.contains(namespace)) { + builder.add(FileTransferDescription.upgrade(child)); + } else if (Namespace.JINGLE_APPS_RTP.equals(namespace)) { + builder.add(RtpDescription.upgrade(child)); + } else { + builder.add(GenericDescription.upgrade(child)); + } + } + } + return builder.build(); + } + + public static Propose upgrade(final Element element) { + Preconditions.checkArgument("propose".equals(element.getName())); + Preconditions.checkArgument(Namespace.JINGLE_MESSAGE.equals(element.getNamespace())); + final Propose propose = new Propose(); + propose.setAttributes(element.getAttributes()); + propose.setChildren(element.getChildren()); + return propose; + } +} diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java index 3351e9301..70f3f0f6a 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java @@ -14,6 +14,7 @@ import java.util.Map; import eu.siacs.conversations.xml.Element; import eu.siacs.conversations.xml.Namespace; +import eu.siacs.conversations.xmpp.jingle.Media; import eu.siacs.conversations.xmpp.jingle.SessionDescription; public class RtpDescription extends GenericDescription { @@ -509,23 +510,6 @@ public class RtpDescription extends GenericDescription { } } - public enum Media { - VIDEO, AUDIO, UNKNOWN; - - @Override - public String toString() { - return super.toString().toLowerCase(Locale.ROOT); - } - - public static Media of(String value) { - try { - return value == null ? UNKNOWN : Media.valueOf(value.toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { - return UNKNOWN; - } - } - } - public static RtpDescription of(final SessionDescription.Media media) { final RtpDescription rtpDescription = new RtpDescription(media.media); final Map> parameterMap = new HashMap<>();