mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-06-03 01:00:04 +02:00
Compare commits
3 commits
5184e8b0ea
...
c044923bd4
Author | SHA1 | Date | |
---|---|---|---|
c044923bd4 | |||
0dea68783b | |||
ccae573cb1 |
|
@ -0,0 +1,50 @@
|
||||||
|
package seaweedfs.client;
|
||||||
|
|
||||||
|
import com.google.protobuf.ByteString;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A tool class for uniformly formatting the keys of Entry's ExtendedMap <br>@date 2024/4/9 10:44<br>
|
||||||
|
* Process "Seaweed-" prefix, consistent with http upload file situation
|
||||||
|
* <br>
|
||||||
|
*
|
||||||
|
* <b>Premise:</b>
|
||||||
|
* <br>
|
||||||
|
* curl -H "Seaweed-name1: value1" -F file=to/path "http://localhost:8888/"
|
||||||
|
* <br>
|
||||||
|
* When uploading files using Http, you must add the "Seaweed-" prefix to the key of Extended. As shown above,
|
||||||
|
* the final storage result is Seaweed-name1, The name of key that actual users understand should be name1
|
||||||
|
* <br>
|
||||||
|
* The key of Extended is not forced to add the "Seaweed-" prefix when uploading files using FilerClient.
|
||||||
|
* This causes inconsistency in the key of Extended format of the two file upload methods.
|
||||||
|
* <br>
|
||||||
|
* <b>solution:</b>
|
||||||
|
* When storing Entry, add the "Seaweed-" prefix to the Extended key,
|
||||||
|
* and remove the "Seaweed-" prefix when reading Entry information.
|
||||||
|
* Users will not be aware of the "Seaweed-" prefix when using it,
|
||||||
|
* and the format of the Extended key in the two upload methods will be unified.
|
||||||
|
*
|
||||||
|
* @author stillmoon
|
||||||
|
*/
|
||||||
|
public class ExtendedFormatUtil {
|
||||||
|
|
||||||
|
public static void addKeyPrefix(FilerProto.Entry.Builder entry) {
|
||||||
|
Map<String, ByteString> extendedMap = new HashMap<>(entry.getExtendedCount());
|
||||||
|
entry.getExtendedMap().forEach((key, val) -> {
|
||||||
|
extendedMap.put("Seaweed-" + key, val);
|
||||||
|
});
|
||||||
|
entry.clearExtended();
|
||||||
|
entry.putAllExtended(extendedMap);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void removeKeyPrefix(FilerProto.Entry.Builder entry) {
|
||||||
|
Map<String, ByteString> extendedMap = new HashMap<>(entry.getExtendedCount());
|
||||||
|
entry.getExtendedMap().forEach((key, val) -> {
|
||||||
|
extendedMap.put(key.replace("Seaweed-", ""), val);
|
||||||
|
});
|
||||||
|
entry.clearExtended();
|
||||||
|
entry.putAllExtended(extendedMap);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -61,6 +61,9 @@ public class FilerClient extends FilerGrpcClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static FilerProto.Entry afterEntryDeserialization(FilerProto.Entry entry) {
|
public static FilerProto.Entry afterEntryDeserialization(FilerProto.Entry entry) {
|
||||||
|
FilerProto.Entry.Builder builder = entry.toBuilder();
|
||||||
|
ExtendedFormatUtil.removeKeyPrefix(builder);
|
||||||
|
entry = builder.build();
|
||||||
if (entry.getChunksList().size() <= 0) {
|
if (entry.getChunksList().size() <= 0) {
|
||||||
if (entry.getContent().isEmpty()) {
|
if (entry.getContent().isEmpty()) {
|
||||||
return entry;
|
return entry;
|
||||||
|
|
|
@ -113,6 +113,7 @@ public class SeaweedWrite {
|
||||||
List<FilerProto.FileChunk> chunks = FileChunkManifest.maybeManifestize(filerClient, entry.getChunksList(), parentDirectory);
|
List<FilerProto.FileChunk> chunks = FileChunkManifest.maybeManifestize(filerClient, entry.getChunksList(), parentDirectory);
|
||||||
entry.clearChunks();
|
entry.clearChunks();
|
||||||
entry.addAllChunks(chunks);
|
entry.addAllChunks(chunks);
|
||||||
|
ExtendedFormatUtil.addKeyPrefix(entry);
|
||||||
filerClient.getBlockingStub().createEntry(
|
filerClient.getBlockingStub().createEntry(
|
||||||
FilerProto.CreateEntryRequest.newBuilder()
|
FilerProto.CreateEntryRequest.newBuilder()
|
||||||
.setDirectory(parentDirectory)
|
.setDirectory(parentDirectory)
|
||||||
|
|
|
@ -1,12 +1,24 @@
|
||||||
package seaweedfs.client;
|
package seaweedfs.client;
|
||||||
|
|
||||||
|
import com.google.protobuf.ByteString;
|
||||||
|
import java.awt.image.DataBuffer;
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
import seaweedfs.client.FilerProto.Entry;
|
||||||
|
import seaweedfs.client.FilerProto.FuseAttributes;
|
||||||
|
|
||||||
public class SeaweedFilerTest {
|
public class SeaweedFilerTest {
|
||||||
|
private static FilerClient filerClient = new FilerClient("localhost", 18888);
|
||||||
public static void main(String[] args){
|
public static void main(String[] args){
|
||||||
|
|
||||||
FilerClient filerClient = new FilerClient("localhost", 18888);
|
|
||||||
|
|
||||||
List<FilerProto.Entry> entries = filerClient.listEntries("/");
|
List<FilerProto.Entry> entries = filerClient.listEntries("/");
|
||||||
|
|
||||||
for (FilerProto.Entry entry : entries) {
|
for (FilerProto.Entry entry : entries) {
|
||||||
|
@ -29,4 +41,49 @@ public class SeaweedFilerTest {
|
||||||
System.out.println("/ should exists");
|
System.out.println("/ should exists");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExtendedKey() throws IOException {
|
||||||
|
String content = "hello";
|
||||||
|
String location = "/hello.txt";
|
||||||
|
HashMap<String, String> metadata = new HashMap<>();
|
||||||
|
// The storage format of key is Seaweed-mime, and the reading format is mime.
|
||||||
|
metadata.put("mime", "text/plain");
|
||||||
|
// The user-specified key "Seaweed-" prefix does not take effect
|
||||||
|
// The storage format of key is Seaweed-name1, and the reading format is name1
|
||||||
|
metadata.put("Seaweed-name1", "value1");
|
||||||
|
|
||||||
|
long ts = System.currentTimeMillis() / 1000;
|
||||||
|
String dir = SeaweedOutputStream.getParentDirectory(location);
|
||||||
|
String name = SeaweedOutputStream.getFileName(location);
|
||||||
|
FuseAttributes.Builder attributes = FuseAttributes.newBuilder()
|
||||||
|
.setMtime(ts)
|
||||||
|
.setCrtime(ts)
|
||||||
|
.setFileMode(755);
|
||||||
|
Entry.Builder entry = Entry.newBuilder()
|
||||||
|
.setName(name)
|
||||||
|
.setIsDirectory(false)
|
||||||
|
.setAttributes(attributes.build());
|
||||||
|
// put metadata
|
||||||
|
for (Map.Entry<String, String> metadataEntry : metadata.entrySet()) {
|
||||||
|
entry.putExtended(metadataEntry.getKey(),
|
||||||
|
ByteString.copyFromUtf8(metadataEntry.getValue()));
|
||||||
|
}
|
||||||
|
try (SeaweedOutputStream outputStream = new SeaweedOutputStream(filerClient, location, entry, 0,
|
||||||
|
1024, "")) {
|
||||||
|
outputStream.write(content.getBytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Entry getEntry = filerClient.lookupEntry(dir, name);
|
||||||
|
Assert.assertNotNull(getEntry);
|
||||||
|
Map<String, ByteString> extendedMap = getEntry.getExtendedMap();
|
||||||
|
HashSet<String> expectKeys = new HashSet<String>() {{
|
||||||
|
add("mime");
|
||||||
|
add("name1");
|
||||||
|
}};
|
||||||
|
Assert.assertEquals(expectKeys, extendedMap.keySet());
|
||||||
|
|
||||||
|
filerClient.deleteEntry(dir, name, true, false, true);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue