Skip to content

Commit b92d0bb

Browse files
luckolucko
committed
Add bulk delete endpoint
1 parent a820389 commit b92d0bb

15 files changed

+192
-83
lines changed

‎src/main/java/me/lucko/bytebin/Bytebin.java

+7-8
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,11 @@
2525

2626
package me.lucko.bytebin;
2727

28+
import com.google.common.collect.ImmutableSet;
2829
import com.google.common.util.concurrent.ThreadFactoryBuilder;
29-
30+
import io.jooby.ExecutionMode;
31+
import io.jooby.Jooby;
32+
import io.prometheus.client.hotspot.DefaultExports;
3033
import me.lucko.bytebin.content.Content;
3134
import me.lucko.bytebin.content.ContentIndexDatabase;
3235
import me.lucko.bytebin.content.ContentLoader;
@@ -45,16 +48,11 @@
4548
import me.lucko.bytebin.util.RateLimitHandler;
4649
import me.lucko.bytebin.util.RateLimiter;
4750
import me.lucko.bytebin.util.TokenGenerator;
48-
4951
import org.apache.logging.log4j.Level;
5052
import org.apache.logging.log4j.LogManager;
5153
import org.apache.logging.log4j.Logger;
5254
import org.apache.logging.log4j.io.IoBuilder;
5355

54-
import io.jooby.ExecutionMode;
55-
import io.jooby.Jooby;
56-
import io.prometheus.client.hotspot.DefaultExports;
57-
5856
import java.nio.file.Paths;
5957
import java.util.ArrayList;
6058
import java.util.List;
@@ -161,7 +159,7 @@ public Bytebin(Configuration config) throws Exception {
161159
config.getString(Option.HOST, "0.0.0.0"),
162160
config.getInt(Option.PORT, 8080),
163161
metrics,
164-
new RateLimitHandler(config.getStringList(Option.API_KEYS)),
162+
new RateLimitHandler(config.getStringList(Option.RATELIMIT_API_KEYS)),
165163
new RateLimiter(
166164
// by default, allow posts at a rate of 30 times every 10 minutes (every 20s)
167165
config.getInt(Option.POST_RATE_LIMIT_PERIOD, 10),
@@ -180,7 +178,8 @@ public Bytebin(Configuration config) throws Exception {
180178
new TokenGenerator(config.getInt(Option.KEY_LENGTH, 7)),
181179
(Content.MEGABYTE_LENGTH * config.getInt(Option.MAX_CONTENT_LENGTH, 10)),
182180
expiryHandler,
183-
config.getStringMap(Option.HTTP_HOST_ALIASES)
181+
config.getStringMap(Option.HTTP_HOST_ALIASES),
182+
ImmutableSet.copyOf(config.getStringList(Option.ADMIN_API_KEYS))
184183
));
185184
this.server.start();
186185

‎src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java

+1-4
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,11 @@
3232
import com.j256.ormlite.jdbc.JdbcConnectionSource;
3333
import com.j256.ormlite.support.ConnectionSource;
3434
import com.j256.ormlite.table.TableUtils;
35-
35+
import io.prometheus.client.Gauge;
3636
import me.lucko.bytebin.content.storage.StorageBackend;
37-
3837
import org.apache.logging.log4j.LogManager;
3938
import org.apache.logging.log4j.Logger;
4039

41-
import io.prometheus.client.Gauge;
42-
4340
import java.io.IOException;
4441
import java.nio.file.Files;
4542
import java.nio.file.Paths;

‎src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java

+52-21
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,14 @@
2727

2828
import com.github.benmanes.caffeine.cache.CacheLoader;
2929
import com.google.common.collect.ImmutableMap;
30-
30+
import io.prometheus.client.Counter;
3131
import me.lucko.bytebin.content.storage.StorageBackend;
32-
3332
import org.apache.logging.log4j.LogManager;
3433
import org.apache.logging.log4j.Logger;
3534
import org.checkerframework.checker.nullness.qual.NonNull;
3635

37-
import io.prometheus.client.Counter;
38-
3936
import java.util.Collection;
37+
import java.util.List;
4038
import java.util.Map;
4139
import java.util.concurrent.Executor;
4240
import java.util.concurrent.ScheduledExecutorService;
@@ -147,6 +145,35 @@ public void save(Content content) {
147145
}
148146
}
149147

148+
/**
149+
* Delete content.
150+
*
151+
* @param content the content to delete
152+
*/
153+
public void delete(Content content) {
154+
String key = content.getKey();
155+
156+
// find the backend that the content is stored in
157+
String backendId = content.getBackendId();
158+
StorageBackend backend = this.backends.get(backendId);
159+
if (backend == null) {
160+
LOGGER.error("[STORAGE] Unable to delete " + key + " - no such backend '" + backendId + "'");
161+
return;
162+
}
163+
164+
// delete the data from the backend
165+
try {
166+
backend.delete(key);
167+
} catch (Exception e) {
168+
LOGGER.warn("[STORAGE] Unable to delete '" + key + "' from the '" + backend.getBackendId() + "' backend", e);
169+
}
170+
171+
// remove the entry from the index
172+
this.index.remove(key);
173+
174+
LOGGER.info("[STORAGE] Deleted '" + key + "' from the '" + backendId + "' backend");
175+
}
176+
150177
/**
151178
* Invalidates/deletes any expired content and updates the metrics gauges
152179
*/
@@ -155,31 +182,35 @@ public void runInvalidationAndRecordMetrics() {
155182
Collection<Content> expired = this.index.getExpired();
156183

157184
for (Content metadata : expired) {
158-
String key = metadata.getKey();
185+
delete(metadata);
186+
}
159187

160-
// find the backend that the content is stored in
161-
String backendId = metadata.getBackendId();
162-
StorageBackend backend = this.backends.get(backendId);
163-
if (backend == null) {
164-
LOGGER.error("[STORAGE] Unable to delete " + key + " - no such backend '" + backendId + "'");
165-
continue;
166-
}
188+
// update metrics
189+
this.index.recordMetrics();
190+
}
167191

168-
// delete the data from the backend
169-
try {
170-
backend.delete(key);
171-
} catch (Exception e) {
172-
LOGGER.warn("[STORAGE] Unable to delete '" + key + "' from the '" + backend.getBackendId() + "' backend", e);
192+
/**
193+
* Bulk deletes the provided keys
194+
*
195+
* @param keys the keys to delete
196+
* @return how many entries were actually deleted
197+
*/
198+
public int bulkDelete(List<String> keys) {
199+
int count = 0;
200+
for (String key : keys) {
201+
Content content = this.index.get(key);
202+
if (content == null) {
203+
continue;
173204
}
174205

175-
// remove the entry from the index
176-
this.index.remove(key);
177-
178-
LOGGER.info("[STORAGE] Deleted '" + key + "' from the '" + backendId + "' backend");
206+
delete(content);
207+
count++;
179208
}
180209

181210
// update metrics
182211
this.index.recordMetrics();
212+
213+
return count;
183214
}
184215

185216
public Executor getExecutor() {

‎src/main/java/me/lucko/bytebin/content/storage/AuditTask.java

-2
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,7 @@
2525

2626
package me.lucko.bytebin.content.storage;
2727

28-
import me.lucko.bytebin.content.Content;
2928
import me.lucko.bytebin.content.ContentIndexDatabase;
30-
3129
import org.apache.logging.log4j.LogManager;
3230
import org.apache.logging.log4j.Logger;
3331

‎src/main/java/me/lucko/bytebin/content/storage/LocalDiskBackend.java

-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727

2828
import me.lucko.bytebin.content.Content;
2929
import me.lucko.bytebin.util.ContentEncoding;
30-
3130
import org.apache.logging.log4j.LogManager;
3231
import org.apache.logging.log4j.Logger;
3332

‎src/main/java/me/lucko/bytebin/content/storage/S3Backend.java

-2
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,8 @@
2626
package me.lucko.bytebin.content.storage;
2727

2828
import me.lucko.bytebin.content.Content;
29-
3029
import org.apache.logging.log4j.LogManager;
3130
import org.apache.logging.log4j.Logger;
32-
3331
import software.amazon.awssdk.core.ResponseInputStream;
3432
import software.amazon.awssdk.core.sync.RequestBody;
3533
import software.amazon.awssdk.services.s3.S3Client;

‎src/main/java/me/lucko/bytebin/http/BytebinServer.java

+16-12
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,6 @@
2525

2626
package me.lucko.bytebin.http;
2727

28-
import me.lucko.bytebin.Bytebin;
29-
import me.lucko.bytebin.content.ContentLoader;
30-
import me.lucko.bytebin.content.ContentStorageHandler;
31-
import me.lucko.bytebin.util.ExpiryHandler;
32-
import me.lucko.bytebin.util.RateLimitHandler;
33-
import me.lucko.bytebin.util.RateLimiter;
34-
import me.lucko.bytebin.util.TokenGenerator;
35-
36-
import org.apache.logging.log4j.LogManager;
37-
import org.apache.logging.log4j.Logger;
38-
3928
import io.jooby.AssetHandler;
4029
import io.jooby.AssetSource;
4130
import io.jooby.Context;
@@ -48,9 +37,20 @@
4837
import io.jooby.StatusCode;
4938
import io.jooby.exception.StatusCodeException;
5039
import io.prometheus.client.Counter;
40+
import me.lucko.bytebin.Bytebin;
41+
import me.lucko.bytebin.content.ContentLoader;
42+
import me.lucko.bytebin.content.ContentStorageHandler;
43+
import me.lucko.bytebin.http.admin.BulkDeleteHandler;
44+
import me.lucko.bytebin.util.ExpiryHandler;
45+
import me.lucko.bytebin.util.RateLimitHandler;
46+
import me.lucko.bytebin.util.RateLimiter;
47+
import me.lucko.bytebin.util.TokenGenerator;
48+
import org.apache.logging.log4j.LogManager;
49+
import org.apache.logging.log4j.Logger;
5150

5251
import java.time.Duration;
5352
import java.util.Map;
53+
import java.util.Set;
5454
import java.util.concurrent.CompletionException;
5555

5656
public class BytebinServer extends Jooby {
@@ -64,7 +64,7 @@ public class BytebinServer extends Jooby {
6464
.labelNames("method", "useragent")
6565
.register();
6666

67-
public BytebinServer(ContentStorageHandler storageHandler, ContentLoader contentLoader, String host, int port, boolean metrics, RateLimitHandler rateLimitHandler, RateLimiter postRateLimiter, RateLimiter putRateLimiter, RateLimiter readRateLimiter, TokenGenerator contentTokenGenerator, long maxContentLength, ExpiryHandler expiryHandler, Map<String, String> hostAliases) {
67+
public BytebinServer(ContentStorageHandler storageHandler, ContentLoader contentLoader, String host, int port, boolean metrics, RateLimitHandler rateLimitHandler, RateLimiter postRateLimiter, RateLimiter putRateLimiter, RateLimiter readRateLimiter, TokenGenerator contentTokenGenerator, long maxContentLength, ExpiryHandler expiryHandler, Map<String, String> hostAliases, Set<String> adminApiKeys) {
6868
ServerOptions serverOpts = new ServerOptions();
6969
serverOpts.setHost(host);
7070
serverOpts.setPort(port);
@@ -136,6 +136,10 @@ public BytebinServer(ContentStorageHandler storageHandler, ContentLoader content
136136
get("/{id:[a-zA-Z0-9]+}", new GetHandler(this, readRateLimiter, rateLimitHandler, contentLoader));
137137
put("/{id:[a-zA-Z0-9]+}", new PutHandler(this, putRateLimiter, rateLimitHandler, storageHandler, contentLoader, maxContentLength, expiryHandler));
138138
});
139+
140+
routes(() -> {
141+
post("/admin/bulkdelete", new BulkDeleteHandler(this, storageHandler, adminApiKeys));
142+
});
139143
}
140144

141145
public static String getMetricsLabel(Context ctx) {

‎src/main/java/me/lucko/bytebin/http/GetHandler.java

+6-10
Original file line numberDiff line numberDiff line change
@@ -25,30 +25,26 @@
2525

2626
package me.lucko.bytebin.http;
2727

28+
import io.jooby.Context;
29+
import io.jooby.MediaType;
30+
import io.jooby.Route;
31+
import io.jooby.StatusCode;
32+
import io.jooby.exception.StatusCodeException;
2833
import me.lucko.bytebin.content.ContentLoader;
2934
import me.lucko.bytebin.util.ContentEncoding;
3035
import me.lucko.bytebin.util.Gzip;
3136
import me.lucko.bytebin.util.RateLimitHandler;
3237
import me.lucko.bytebin.util.RateLimiter;
3338
import me.lucko.bytebin.util.TokenGenerator;
34-
3539
import org.apache.logging.log4j.LogManager;
3640
import org.apache.logging.log4j.Logger;
3741

38-
import io.jooby.Context;
39-
import io.jooby.MediaType;
40-
import io.jooby.Route;
41-
import io.jooby.StatusCode;
42-
import io.jooby.exception.StatusCodeException;
43-
42+
import javax.annotation.Nonnull;
4443
import java.io.IOException;
4544
import java.time.Instant;
4645
import java.util.List;
4746
import java.util.Set;
4847
import java.util.concurrent.CompletableFuture;
49-
import java.util.stream.Collectors;
50-
51-
import javax.annotation.Nonnull;
5248

5349
public final class GetHandler implements Route.Handler {
5450

‎src/main/java/me/lucko/bytebin/http/MetricsHandler.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,8 @@
3232
import io.prometheus.client.CollectorRegistry;
3333
import io.prometheus.client.exporter.common.TextFormat;
3434

35-
import java.io.OutputStreamWriter;
36-
3735
import javax.annotation.Nonnull;
36+
import java.io.OutputStreamWriter;
3837

3938
public final class MetricsHandler implements Route.Handler {
4039

‎src/main/java/me/lucko/bytebin/http/PostHandler.java

+7-10
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@
2525

2626
package me.lucko.bytebin.http;
2727

28+
import io.jooby.Context;
29+
import io.jooby.MediaType;
30+
import io.jooby.Route;
31+
import io.jooby.StatusCode;
32+
import io.jooby.exception.StatusCodeException;
33+
import io.prometheus.client.Summary;
2834
import me.lucko.bytebin.content.Content;
2935
import me.lucko.bytebin.content.ContentLoader;
3036
import me.lucko.bytebin.content.ContentStorageHandler;
@@ -34,24 +40,15 @@
3440
import me.lucko.bytebin.util.RateLimitHandler;
3541
import me.lucko.bytebin.util.RateLimiter;
3642
import me.lucko.bytebin.util.TokenGenerator;
37-
3843
import org.apache.logging.log4j.LogManager;
3944
import org.apache.logging.log4j.Logger;
4045

41-
import io.jooby.Context;
42-
import io.jooby.MediaType;
43-
import io.jooby.Route;
44-
import io.jooby.StatusCode;
45-
import io.jooby.exception.StatusCodeException;
46-
import io.prometheus.client.Summary;
47-
46+
import javax.annotation.Nonnull;
4847
import java.util.Date;
4948
import java.util.List;
5049
import java.util.Map;
5150
import java.util.concurrent.CompletableFuture;
5251

53-
import javax.annotation.Nonnull;
54-
5552
public final class PostHandler implements Route.Handler {
5653

5754
/** Logger instance */

‎src/main/java/me/lucko/bytebin/http/PutHandler.java

+5-8
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@
2525

2626
package me.lucko.bytebin.http;
2727

28+
import io.jooby.Context;
29+
import io.jooby.Route;
30+
import io.jooby.StatusCode;
31+
import io.jooby.exception.StatusCodeException;
2832
import me.lucko.bytebin.content.ContentLoader;
2933
import me.lucko.bytebin.content.ContentStorageHandler;
3034
import me.lucko.bytebin.util.ContentEncoding;
@@ -33,22 +37,15 @@
3337
import me.lucko.bytebin.util.RateLimitHandler;
3438
import me.lucko.bytebin.util.RateLimiter;
3539
import me.lucko.bytebin.util.TokenGenerator;
36-
3740
import org.apache.logging.log4j.LogManager;
3841
import org.apache.logging.log4j.Logger;
3942

40-
import io.jooby.Context;
41-
import io.jooby.Route;
42-
import io.jooby.StatusCode;
43-
import io.jooby.exception.StatusCodeException;
44-
43+
import javax.annotation.Nonnull;
4544
import java.util.Date;
4645
import java.util.List;
4746
import java.util.concurrent.CompletableFuture;
4847
import java.util.concurrent.atomic.AtomicReference;
4948

50-
import javax.annotation.Nonnull;
51-
5249
public final class PutHandler implements Route.Handler {
5350

5451
/** Logger instance */

0 commit comments

Comments
 (0)