PeterCxy/cheogram
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

treat transport-info w/o candidates and changed credentials as offer

Browse source
This commit is contained in:
Daniel Gultsch 2021-11-14 18:22:18 +01:00
parent 717c83753f
commit 5b80c62a63
5 changed files with 254 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/main/java/eu/siacs/conversations/xml/Element.java
View file

@ -1,5 +1,7 @@
package eu.siacs.conversations.xml;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.Hashtable;
import java.util.List;
@ -165,8 +167,9 @@ public class Element {
return this.attributes;
}
@NotNull
public String toString() {
StringBuilder elementOutput = new StringBuilder();
final StringBuilder elementOutput = new StringBuilder();
if ((content == null) && (children.size() == 0)) {
Tag emptyTag = Tag.empty(name);
emptyTag.setAtttributes(this.attributes);

193
src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java
View file

@ -25,7 +25,6 @@ import org.webrtc.IceCandidate;
import org.webrtc.PeerConnection;
import org.webrtc.VideoTrack;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@ -142,7 +141,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this);
private final ArrayDeque<Set<Map.Entry<String, RtpContentMap.DescriptionTransport>>> pendingIceCandidates = new ArrayDeque<>();
//TODO convert to Queue<Map.Entry<String, Description>>?
private final Queue<Map.Entry<String, RtpContentMap.DescriptionTransport>> pendingIceCandidates = new LinkedList<>();
private final OmemoVerification omemoVerification = new OmemoVerification();
private final Message message;
private State state = State.NULL;
@ -193,7 +193,6 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
@Override
synchronized void deliverPacket(final JinglePacket jinglePacket) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection");
switch (jinglePacket.getAction()) {
case SESSION_INITIATE:
receiveSessionInitiate(jinglePacket);
@ -254,23 +253,29 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
private void receiveTransportInfo(final JinglePacket jinglePacket) {
//Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received
if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) {
respondOk(jinglePacket);
final RtpContentMap contentMap;
try {
contentMap = RtpContentMap.of(jinglePacket);
} catch (IllegalArgumentException | NullPointerException e) {
} catch (final IllegalArgumentException | NullPointerException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e);
respondOk(jinglePacket);
return;
}
final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
if (this.state == State.SESSION_ACCEPTED) {
//zero candidates + modified credentials are an ICE restart offer
if (checkForIceRestart(contentMap, jinglePacket)) {
return;
}
respondOk(jinglePacket);
try {
processCandidates(candidates);
} catch (final WebRTCWrapper.PeerConnectionNotInitialized e) {
Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored");
}
} else {
pendingIceCandidates.push(candidates);
respondOk(jinglePacket);
pendingIceCandidates.addAll(candidates);
}
} else {
if (isTerminated()) {
@ -283,37 +288,106 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
}
private boolean checkForIceRestart(final RtpContentMap rtpContentMap, final JinglePacket jinglePacket) {
final RtpContentMap existing = getRemoteContentMap();
final Map<String, IceUdpTransportInfo.Credentials> existingCredentials = existing.getCredentials();
final Map<String, IceUdpTransportInfo.Credentials> newCredentials = rtpContentMap.getCredentials();
if (!existingCredentials.keySet().equals(newCredentials.keySet())) {
return false;
}
if (existingCredentials.equals(newCredentials)) {
return false;
}
final boolean isOffer = rtpContentMap.emptyCandidates();
Log.d(Config.LOGTAG, "detected ICE restart. offer=" + isOffer + " " + jinglePacket);
//TODO reset to 'actpass'?
final RtpContentMap restartContentMap = existing.modifiedCredentials(newCredentials);
try {
if (applyIceRestart(isOffer, restartContentMap)) {
return false;
} else {
Log.d(Config.LOGTAG, "responding with tie break");
//TODO respond with conflict
return true;
}
} catch (Exception e) {
Log.d(Config.LOGTAG, "failure to apply ICE restart. sending error", e);
//TODO send some kind of error
return true;
}
}
private boolean applyIceRestart(final boolean isOffer, final RtpContentMap restartContentMap) throws ExecutionException, InterruptedException {
final SessionDescription sessionDescription = SessionDescription.of(restartContentMap);
final org.webrtc.SessionDescription.Type type = isOffer ? org.webrtc.SessionDescription.Type.OFFER : org.webrtc.SessionDescription.Type.ANSWER;
org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription(type, sessionDescription.toString());
if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) {
if (isInitiator()) {
//We ignore the offer and respond with tie-break. This will clause the responder not to apply the content map
return false;
}
//rollback our own local description. should happen automatically but doesn't
webRTCWrapper.rollbackLocalDescription().get();
}
webRTCWrapper.setRemoteDescription(sdp).get();
if (isInitiator()) {
this.responderRtpContentMap = restartContentMap;
} else {
this.initiatorRtpContentMap = restartContentMap;
}
if (isOffer) {
webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
final SessionDescription localSessionDescription = setLocalSessionDescription();
setLocalContentMap(RtpContentMap.of(localSessionDescription));
webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
}
return true;
}
private void processCandidates(final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
processCandidate(content);
}
}
private void processCandidate(final Map.Entry<String, RtpContentMap.DescriptionTransport> content) {
final RtpContentMap rtpContentMap = getRemoteContentMap();
final List<String> indices = toIdentificationTags(rtpContentMap);
final String sdpMid = content.getKey(); //aka content name
final IceUdpTransportInfo transport = content.getValue().transport;
final IceUdpTransportInfo.Credentials credentials = transport.getCredentials();
//TODO check that credentials remained the same
for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) {
final String sdp;
try {
sdp = candidate.toSdpAttribute(credentials.ufrag);
} catch (final IllegalArgumentException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
continue;
}
final int mLineIndex = indices.indexOf(sdpMid);
if (mLineIndex < 0) {
Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
}
final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
this.webRTCWrapper.addIceCandidate(iceCandidate);
}
}
private RtpContentMap getRemoteContentMap() {
return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap;
}
private List<String> toIdentificationTags(final RtpContentMap rtpContentMap) {
final Group originalGroup = rtpContentMap.group;
final List<String> identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags();
if (identificationTags.size() == 0) {
Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices");
}
processCandidates(identificationTags, contents);
}
private void processCandidates(final List<String> indices, final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> contents) {
for (final Map.Entry<String, RtpContentMap.DescriptionTransport> content : contents) {
final String ufrag = content.getValue().transport.getAttribute("ufrag");
for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) {
final String sdp;
try {
sdp = candidate.toSdpAttribute(ufrag);
} catch (IllegalArgumentException e) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage());
continue;
}
final String sdpMid = content.getKey();
final int mLineIndex = indices.indexOf(sdpMid);
if (mLineIndex < 0) {
Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices);
}
final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp);
Log.d(Config.LOGTAG, "received candidate: " + iceCandidate);
this.webRTCWrapper.addIceCandidate(iceCandidate);
}
}
return identificationTags;
}
private ListenableFuture<RtpContentMap> receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) {
@ -401,11 +475,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) {
respondOk(jinglePacket);
final Set<Map.Entry<String, RtpContentMap.DescriptionTransport>> candidates = contentMap.contents.entrySet();
if (candidates.size() > 0) {
pendingIceCandidates.push(candidates);
}
pendingIceCandidates.addAll(contentMap.contents.entrySet());
if (target == State.SESSION_INITIALIZED_PRE_APPROVED) {
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate");
sendSessionAccept();
@ -495,8 +565,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
sendSessionTerminate(Reason.FAILED_APPLICATION);
return;
}
final List<String> identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags();
processCandidates(identificationTags, contentMap.contents.entrySet());
processCandidates(contentMap.contents.entrySet());
}
private void sendSessionAccept() {
@ -558,9 +627,10 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
}
private void addIceCandidatesFromBlackLog() {
while (!this.pendingIceCandidates.isEmpty()) {
processCandidates(this.pendingIceCandidates.poll());
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log");
Map.Entry<String, RtpContentMap.DescriptionTransport> foo;
while ((foo = this.pendingIceCandidates.poll()) != null) {
processCandidate(foo);
Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidate from back log");
}
}
@ -1335,7 +1405,13 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
@Override
public void onIceCandidate(final IceCandidate iceCandidate) {
final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp);
final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap;
final Collection<String> currentUfrags = Collections2.transform(rtpContentMap.getCredentials().values(), c -> c.ufrag);
final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, currentUfrags);
if (candidate == null) {
Log.d(Config.LOGTAG,"ignoring (not sending) candidate: "+iceCandidate.toString());
return;
}
Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString());
sendTransportInfo(iceCandidate.sdpMid, candidate);
}
@ -1373,23 +1449,42 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web
@Override
public void onRenegotiationNeeded() {
Log.d(Config.LOGTAG, "onRenegotiationNeeded()");
this.webRTCWrapper.execute(this::renegotiate);
this.webRTCWrapper.execute(this::initiateIceRestart);
}
private void renegotiate() {
private void initiateIceRestart() {
PeerConnection.SignalingState signalingState = webRTCWrapper.getSignalingState();
Log.d(Config.LOGTAG, "initiateIceRestart() - " + signalingState);
if (signalingState != PeerConnection.SignalingState.STABLE) {
return;
}
this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
final SessionDescription sessionDescription;
try {
final SessionDescription sessionDescription = setLocalSessionDescription();
final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
setRenegotiatedContentMap(rtpContentMap);
this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
sessionDescription = setLocalSessionDescription();
} catch (final Exception e) {
Log.d(Config.LOGTAG, "failed to renegotiate", e);
//TODO send some sort of failure (comparable to when initiating)
return;
}
final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription);
final RtpContentMap transportInfo = rtpContentMap.transportInfo();
final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId);
Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket);
jinglePacket.setTo(id.with);
xmppConnectionService.sendIqPacket(id.account, jinglePacket, (account, response) -> {
if (response.getType() == IqPacket.TYPE.RESULT) {
Log.d(Config.LOGTAG, "received success to our ice restart");
setLocalContentMap(rtpContentMap);
webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
} else {
Log.d(Config.LOGTAG, "received failure to our ice restart");
//TODO handle tie-break. Rollback?
}
});
}
private void setRenegotiatedContentMap(final RtpContentMap rtpContentMap) {
private void setLocalContentMap(final RtpContentMap rtpContentMap) {
if (isInitiator()) {
this.initiatorRtpContentMap = rtpContentMap;
} else {

34
src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java
View file

@ -1,7 +1,5 @@
package eu.siacs.conversations.xmpp.jingle;
import android.util.Log;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@ -17,9 +15,9 @@ import org.checkerframework.checker.nullness.compatqual.NullableDecl;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import eu.siacs.conversations.Config;
import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription;
import eu.siacs.conversations.xmpp.jingle.stanzas.GenericTransportInfo;
@ -137,7 +135,37 @@ public class RtpContentMap {
final IceUdpTransportInfo newTransportInfo = transportInfo.cloneWrapper();
newTransportInfo.addChild(candidate);
return new RtpContentMap(null, ImmutableMap.of(contentName, new DescriptionTransport(null, newTransportInfo)));
}
RtpContentMap transportInfo() {
return new RtpContentMap(
null,
Maps.transformValues(contents, dt -> new DescriptionTransport(null, dt.transport.cloneWrapper()))
);
}
public Map<String, IceUdpTransportInfo.Credentials> getCredentials() {
return Maps.transformValues(contents, dt -> dt.transport.getCredentials());
}
public boolean emptyCandidates() {
int count = 0;
for (DescriptionTransport descriptionTransport : contents.values()) {
count += descriptionTransport.transport.getCandidates().size();
}
return count == 0;
}
public RtpContentMap modifiedCredentials(Map<String, IceUdpTransportInfo.Credentials> credentialsMap) {
final ImmutableMap.Builder<String, DescriptionTransport> contentMapBuilder = new ImmutableMap.Builder<>();
for (final Map.Entry<String, DescriptionTransport> content : contents.entrySet()) {
final RtpDescription rtpDescription = content.getValue().description;
IceUdpTransportInfo transportInfo = content.getValue().transport;
final IceUdpTransportInfo.Credentials credentials = Objects.requireNonNull(credentialsMap.get(content.getKey()));
final IceUdpTransportInfo modifiedTransportInfo = transportInfo.modifyCredentials(credentials);
contentMapBuilder.put(content.getKey(), new DescriptionTransport(rtpDescription, modifiedTransportInfo));
}
return new RtpContentMap(this.group, contentMapBuilder.build());
}
public static class DescriptionTransport {

34
src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java
View file

@ -17,7 +17,6 @@ import com.google.common.util.concurrent.SettableFuture;
import org.webrtc.AudioSource;
import org.webrtc.AudioTrack;
import org.webrtc.Camera1Enumerator;
import org.webrtc.Camera2Enumerator;
import org.webrtc.CameraEnumerationAndroid;
import org.webrtc.CameraEnumerator;
@ -87,6 +86,7 @@ public class WebRTCWrapper {
private final EventCallback eventCallback;
private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false);
private final AtomicBoolean ignoreOnRenegotiationNeeded = new AtomicBoolean(false);
private final Queue<IceCandidate> iceCandidates = new LinkedList<>();
private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() {
@Override
@ -163,6 +163,10 @@ public class WebRTCWrapper {
@Override
public void onRenegotiationNeeded() {
if (ignoreOnRenegotiationNeeded.get()) {
Log.d(EXTENDED_LOGGING_TAG, "ignoring onRenegotiationNeeded()");
return;
}
Log.d(EXTENDED_LOGGING_TAG, "onRenegotiationNeeded()");
final PeerConnection.PeerConnectionState currentState = peerConnection == null ? null : peerConnection.connectionState();
if (currentState != null && currentState != PeerConnection.PeerConnectionState.NEW) {
@ -307,12 +311,12 @@ public class WebRTCWrapper {
}
void restartIce() {
executorService.execute(()-> requirePeerConnection().restartIce());
executorService.execute(() -> requirePeerConnection().restartIce());
}
public void setIsReadyToReceiveIceCandidates(final boolean ready) {
readyToReceivedIceCandidates.set(ready);
while(ready && iceCandidates.peek() != null) {
while (ready && iceCandidates.peek() != null) {
eventCallback.onIceCandidate(iceCandidates.poll());
}
}
@ -452,6 +456,26 @@ public class WebRTCWrapper {
}, MoreExecutors.directExecutor());
}
public ListenableFuture<Void> rollbackLocalDescription() {
final SettableFuture<Void> future = SettableFuture.create();
final SessionDescription rollback = new SessionDescription(SessionDescription.Type.ROLLBACK, "");
ignoreOnRenegotiationNeeded.set(true);
requirePeerConnection().setLocalDescription(new SetSdpObserver() {
@Override
public void onSetSuccess() {
future.set(null);
ignoreOnRenegotiationNeeded.set(false);
}
@Override
public void onSetFailure(final String message) {
future.setException(new FailureToSetDescriptionException(message));
}
}, rollback);
return future;
}
private static void logDescription(final SessionDescription sessionDescription) {
for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {
Log.d(EXTENDED_LOGGING_TAG, line);
@ -552,6 +576,10 @@ public class WebRTCWrapper {
executorService.execute(command);
}
public PeerConnection.SignalingState getSignalingState() {
return requirePeerConnection().signalingState();
}
public interface EventCallback {
void onIceCandidate(IceCandidate iceCandidate);

45
src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java
View file

@ -1,6 +1,7 @@
package eu.siacs.conversations.xmpp.jingle.stanzas;
import com.google.common.base.Joiner;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
@ -8,6 +9,7 @@ import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import java.util.Collection;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.LinkedHashMap;
@ -58,6 +60,12 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
return fingerprint == null ? null : Fingerprint.upgrade(fingerprint);
}
public Credentials getCredentials() {
final String ufrag = this.getAttribute("ufrag");
final String password = this.getAttribute("pwd");
return new Credentials(ufrag, password);
}
public List<Candidate> getCandidates() {
final ImmutableList.Builder<Candidate> builder = new ImmutableList.Builder<>();
for (final Element child : getChildren()) {
@ -74,6 +82,37 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
return transportInfo;
}
public IceUdpTransportInfo modifyCredentials(Credentials credentials) {
final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo();
transportInfo.setAttribute("ufrag", credentials.ufrag);
transportInfo.setAttribute("pwd", credentials.password);
transportInfo.setChildren(getChildren());
return transportInfo;
}
public static class Credentials {
public final String ufrag;
public final String password;
public Credentials(String ufrag, String password) {
this.ufrag = ufrag;
this.password = password;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Credentials that = (Credentials) o;
return Objects.equal(ufrag, that.ufrag) && Objects.equal(password, that.password);
}
@Override
public int hashCode() {
return Objects.hashCode(ufrag, password);
}
}
public static class Candidate extends Element {
private Candidate() {
@ -89,7 +128,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
}
// https://tools.ietf.org/html/draft-ietf-mmusic-ice-sip-sdp-39#section-5.1
public static Candidate fromSdpAttribute(final String attribute) {
public static Candidate fromSdpAttribute(final String attribute, Collection<String> currentUfrags) {
final String[] pair = attribute.split(":", 2);
if (pair.length == 2 && "candidate".equals(pair[0])) {
final String[] segments = pair[1].split(" ");
@ -105,6 +144,10 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
for (int i = 6; i < segments.length - 1; i = i + 2) {
additional.put(segments[i], segments[i + 1]);
}
final String ufrag = additional.get("ufrag");
if (ufrag != null && !currentUfrags.contains(ufrag)) {
return null;
}
final Candidate candidate = new Candidate();
candidate.setAttribute("component", component);
candidate.setAttribute("foundation", foundation);