mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-05-09 13:00:45 +02:00
Compare commits
7 commits
c044923bd4
...
5184e8b0ea
Author | SHA1 | Date | |
---|---|---|---|
5184e8b0ea | |||
abf01a0eb7 | |||
855607c536 | |||
607927da60 | |||
2f3fee9bb9 | |||
0dea68783b | |||
ccae573cb1 |
2
go.mod
2
go.mod
|
@ -18,7 +18,6 @@ require (
|
|||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/eapache/go-resiliency v1.3.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
|
||||
|
@ -142,6 +141,7 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/config v1.27.11
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4
|
||||
github.com/cognusion/imaging v1.0.1
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/getsentry/sentry-go v0.27.0
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1
|
||||
|
|
5
go.sum
5
go.sum
|
@ -232,6 +232,8 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH
|
|||
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cognusion/imaging v1.0.1 h1:jJa1+jYHvr2zS5zZxoluYthH5KbVz4LEvD3xy/W2L90=
|
||||
github.com/cognusion/imaging v1.0.1/go.mod h1:ucYm08RsFoQvYXEV5XMsRBppxrWzD1AGxm6iod5/rvM=
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9W0=
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0/go.mod h1:0NAO+/3knbMx6+5pCv+Hcbaz4xn/Zzbn9+WIib2rKVI=
|
||||
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
|
||||
|
@ -252,8 +254,6 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC
|
|||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
|
||||
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
|
||||
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU=
|
||||
|
@ -1074,7 +1074,6 @@ golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu
|
|||
golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
|
||||
golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
|
|
@ -0,0 +1,50 @@
|
|||
package seaweedfs.client;
|
||||
|
||||
import com.google.protobuf.ByteString;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* A tool class for uniformly formatting the keys of Entry's ExtendedMap <br>@date 2024/4/9 10:44<br>
|
||||
* Process "Seaweed-" prefix, consistent with http upload file situation
|
||||
* <br>
|
||||
*
|
||||
* <b>Premise:</b>
|
||||
* <br>
|
||||
* curl -H "Seaweed-name1: value1" -F file=to/path "http://localhost:8888/"
|
||||
* <br>
|
||||
* When uploading files using Http, you must add the "Seaweed-" prefix to the key of Extended. As shown above,
|
||||
* the final storage result is Seaweed-name1, The name of key that actual users understand should be name1
|
||||
* <br>
|
||||
* The key of Extended is not forced to add the "Seaweed-" prefix when uploading files using FilerClient.
|
||||
* This causes inconsistency in the key of Extended format of the two file upload methods.
|
||||
* <br>
|
||||
* <b>solution:</b>
|
||||
* When storing Entry, add the "Seaweed-" prefix to the Extended key,
|
||||
* and remove the "Seaweed-" prefix when reading Entry information.
|
||||
* Users will not be aware of the "Seaweed-" prefix when using it,
|
||||
* and the format of the Extended key in the two upload methods will be unified.
|
||||
*
|
||||
* @author stillmoon
|
||||
*/
|
||||
public class ExtendedFormatUtil {
|
||||
|
||||
public static void addKeyPrefix(FilerProto.Entry.Builder entry) {
|
||||
Map<String, ByteString> extendedMap = new HashMap<>(entry.getExtendedCount());
|
||||
entry.getExtendedMap().forEach((key, val) -> {
|
||||
extendedMap.put("Seaweed-" + key, val);
|
||||
});
|
||||
entry.clearExtended();
|
||||
entry.putAllExtended(extendedMap);
|
||||
}
|
||||
|
||||
public static void removeKeyPrefix(FilerProto.Entry.Builder entry) {
|
||||
Map<String, ByteString> extendedMap = new HashMap<>(entry.getExtendedCount());
|
||||
entry.getExtendedMap().forEach((key, val) -> {
|
||||
extendedMap.put(key.replace("Seaweed-", ""), val);
|
||||
});
|
||||
entry.clearExtended();
|
||||
entry.putAllExtended(extendedMap);
|
||||
}
|
||||
|
||||
}
|
|
@ -61,6 +61,9 @@ public class FilerClient extends FilerGrpcClient {
|
|||
}
|
||||
|
||||
public static FilerProto.Entry afterEntryDeserialization(FilerProto.Entry entry) {
|
||||
FilerProto.Entry.Builder builder = entry.toBuilder();
|
||||
ExtendedFormatUtil.removeKeyPrefix(builder);
|
||||
entry = builder.build();
|
||||
if (entry.getChunksList().size() <= 0) {
|
||||
if (entry.getContent().isEmpty()) {
|
||||
return entry;
|
||||
|
|
|
@ -113,6 +113,7 @@ public class SeaweedWrite {
|
|||
List<FilerProto.FileChunk> chunks = FileChunkManifest.maybeManifestize(filerClient, entry.getChunksList(), parentDirectory);
|
||||
entry.clearChunks();
|
||||
entry.addAllChunks(chunks);
|
||||
ExtendedFormatUtil.addKeyPrefix(entry);
|
||||
filerClient.getBlockingStub().createEntry(
|
||||
FilerProto.CreateEntryRequest.newBuilder()
|
||||
.setDirectory(parentDirectory)
|
||||
|
|
|
@ -1,12 +1,24 @@
|
|||
package seaweedfs.client;
|
||||
|
||||
import com.google.protobuf.ByteString;
|
||||
import java.awt.image.DataBuffer;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import seaweedfs.client.FilerProto.Entry;
|
||||
import seaweedfs.client.FilerProto.FuseAttributes;
|
||||
|
||||
public class SeaweedFilerTest {
|
||||
private static FilerClient filerClient = new FilerClient("localhost", 18888);
|
||||
public static void main(String[] args){
|
||||
|
||||
FilerClient filerClient = new FilerClient("localhost", 18888);
|
||||
|
||||
List<FilerProto.Entry> entries = filerClient.listEntries("/");
|
||||
|
||||
for (FilerProto.Entry entry : entries) {
|
||||
|
@ -29,4 +41,49 @@ public class SeaweedFilerTest {
|
|||
System.out.println("/ should exists");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtendedKey() throws IOException {
|
||||
String content = "hello";
|
||||
String location = "/hello.txt";
|
||||
HashMap<String, String> metadata = new HashMap<>();
|
||||
// The storage format of key is Seaweed-mime, and the reading format is mime.
|
||||
metadata.put("mime", "text/plain");
|
||||
// The user-specified key "Seaweed-" prefix does not take effect
|
||||
// The storage format of key is Seaweed-name1, and the reading format is name1
|
||||
metadata.put("Seaweed-name1", "value1");
|
||||
|
||||
long ts = System.currentTimeMillis() / 1000;
|
||||
String dir = SeaweedOutputStream.getParentDirectory(location);
|
||||
String name = SeaweedOutputStream.getFileName(location);
|
||||
FuseAttributes.Builder attributes = FuseAttributes.newBuilder()
|
||||
.setMtime(ts)
|
||||
.setCrtime(ts)
|
||||
.setFileMode(755);
|
||||
Entry.Builder entry = Entry.newBuilder()
|
||||
.setName(name)
|
||||
.setIsDirectory(false)
|
||||
.setAttributes(attributes.build());
|
||||
// put metadata
|
||||
for (Map.Entry<String, String> metadataEntry : metadata.entrySet()) {
|
||||
entry.putExtended(metadataEntry.getKey(),
|
||||
ByteString.copyFromUtf8(metadataEntry.getValue()));
|
||||
}
|
||||
try (SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, location, entry, 0,
|
||||
1024, "")) {
|
||||
outputStream.write(content.getBytes());
|
||||
}
|
||||
|
||||
|
||||
Entry getEntry = filerClient.lookupEntry(dir, name);
|
||||
Assert.assertNotNull(getEntry);
|
||||
Map<String, ByteString> extendedMap = getEntry.getExtendedMap();
|
||||
HashSet<String> expectKeys = new HashSet<String>() {{
|
||||
add("mime");
|
||||
add("name1");
|
||||
}};
|
||||
Assert.assertEquals(expectKeys, extendedMap.keySet());
|
||||
|
||||
filerClient.deleteEntry(dir, name, true, false, true);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -72,7 +72,9 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
|
|||
return servers[i] < servers[j]
|
||||
})
|
||||
|
||||
r.Lock()
|
||||
r.lastUpdateTime = time.Now()
|
||||
r.Unlock()
|
||||
|
||||
r.addOneSnapshot(servers)
|
||||
|
||||
|
|
|
@ -8,7 +8,7 @@ import (
|
|||
"image/png"
|
||||
"io"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"github.com/cognusion/imaging"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
|
|
@ -8,7 +8,7 @@ import (
|
|||
"image/png"
|
||||
"io"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
"github.com/cognusion/imaging"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
|
||||
|
|
|
@ -328,7 +328,7 @@ func ReadUrlAsStreamAuthenticated(fileUrl, jwt string, cipherKey []byte, isConte
|
|||
}
|
||||
defer CloseResponse(r)
|
||||
if r.StatusCode >= 400 {
|
||||
retryable = r.StatusCode == http.StatusNotFound || r.StatusCode >= 500
|
||||
retryable = r.StatusCode == http.StatusNotFound || r.StatusCode >= 499
|
||||
return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status)
|
||||
}
|
||||
|
||||
|
|
|
@ -66,9 +66,17 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition
|
|||
isDone = true
|
||||
return
|
||||
}
|
||||
logBuffer.RLock()
|
||||
lastTsNs := logBuffer.LastTsNs
|
||||
for lastTsNs == logBuffer.LastTsNs {
|
||||
logBuffer.RUnlock()
|
||||
loopTsNs := lastTsNs // make a copy
|
||||
|
||||
for lastTsNs == loopTsNs {
|
||||
if waitForDataFn() {
|
||||
// Update loopTsNs and loop again
|
||||
logBuffer.RLock()
|
||||
loopTsNs = logBuffer.LastTsNs
|
||||
logBuffer.RUnlock()
|
||||
continue
|
||||
} else {
|
||||
isDone = true
|
||||
|
|
Loading…
Reference in a new issue