SpeechToTextServlet.java
package server.restful.servlets;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.AsyncContext;
import jakarta.servlet.annotation.HttpConstraint;
import jakarta.servlet.annotation.ServletSecurity;
import jakarta.servlet.annotation.WebServlet;
import jakarta.servlet.http.HttpFilter;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.servlet.http.HttpSession;
import libsidutils.IOUtils;
import org.vosk.LibVosk;
import org.vosk.LogLevel;
import org.vosk.Model;
import org.vosk.Recognizer;
import server.restful.common.JSIDPlay2Servlet;
import server.restful.common.SessionLockManager;
import server.restful.common.async.DefaultThreadFactory;
import server.restful.common.async.HttpAsyncContextRunnable;
import server.restful.common.converter.LocaleConverter;
import server.restful.common.filters.CounterBasedRateLimiterFilter;
import server.restful.common.parameter.RequestResponseBodyParameter;
import server.restful.common.parameter.RequestResponseBodyParameter.Blob;
import server.restful.common.parameter.ServletParameterParser;
import server.restful.common.parameter.UsageParameter;
import server.restful.common.processor.ExternalProcess;
import server.restful.common.speech2text.AudioBuffer;
import server.restful.common.speech2text.DeepFilterParameters;
import sidplay.audio.wav.WAVHeader;
import ui.entities.config.SidPlay2Section;
import java.io.*;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.*;
import static java.lang.Integer.MIN_VALUE;
import static java.lang.String.format;
import static libsidutils.IOUtils.ioWrap;
import static org.apache.http.HttpStatus.SC_TOO_MANY_REQUESTS;
import static server.restful.JSIDPlay2Server.*;
import static server.restful.common.IServletSystemProperties.SpeechToTextServlet.*;
import static server.restful.common.ServletUtil.error;
import static server.restful.common.filters.CounterBasedRateLimiterFilter.FILTER_PARAMETER_MAX_REQUESTS_PER_SERVLET;
@WebServlet(name = "SpeechToTextServlet", displayName = "SpeechToTextServlet", asyncSupported = true, urlPatterns = CONTEXT_ROOT_SERVLET
+ "/speech2text", description = "Speech recognition")
@ServletSecurity(value = @HttpConstraint(rolesAllowed = {ROLE_USER, ROLE_ADMIN}))
public class SpeechToTextServlet extends JSIDPlay2Servlet {
@SuppressWarnings("unused")
@Parameters(resourceBundle = "server.restful.servlets.SpeechToTextServletParameters")
public static class SpeechToTextServletParameters extends UsageParameter
implements RequestResponseBodyParameter<Blob, String> {
private Locale locale = Locale.ROOT;
public Locale getLocale() {
return locale;
}
@Parameter(names = "--locale", descriptionKey = "LOCALE", converter = LocaleConverter.class, order = MIN_VALUE
+ 1)
public void setLocale(Locale locale) {
this.locale = locale;
}
private Locale speechToTextLocale;
public Locale getSpeechToTextLocale() {
return speechToTextLocale;
}
@Parameter(names = "--speechToTextLocale", descriptionKey = "SPEECH_TO_TEXT_LOCALE", converter = LocaleConverter.class, order = MIN_VALUE
+ 2)
public void setSpeechToTextLocale(Locale speechToTextLocale) {
this.speechToTextLocale = speechToTextLocale;
}
private int samplingRate = 48000;
public int getSamplingRate() {
return samplingRate;
}
@Parameter(names = "--samplingRate", descriptionKey = "FREQUENCY", order = MIN_VALUE + 3)
public void setSamplingRate(int samplingRate) {
this.samplingRate = samplingRate;
}
private List<String> commands;
public List<String> getCommands() {
return commands;
}
@Parameter(names = "--commands", descriptionKey = "COMMANDS", order = MIN_VALUE + 4)
public void setCommands(List<String> commands) {
this.commands = commands;
}
@Override
public Blob getRequestBodyExample() {
return new Blob();
}
@Override
public String getResponseBodyExample() {
return "Hello World";
}
}
private ExecutorService executorService;
private Model modelDe, modelEn;
private File deepFilterOutputDir;
@Override
public void init() {
executorService = new ThreadPoolExecutor(MAX_SPEECHTOTEXT_IN_PARALLEL_CORE, MAX_SPEECHTOTEXT_IN_PARALLEL,
SPEECHTOTEXT_KEEPALIVE_TIME, TimeUnit.SECONDS, new ArrayBlockingQueue<>(SPEECHTOTEXT_QUEUE_SIZE),
new DefaultThreadFactory("/speechtotext"), new ThreadPoolExecutor.AbortPolicy());
try {
LibVosk.setLogLevel(LogLevel.WARNINGS);
modelDe = new Model(SPEECHTOTEXT_MODEL_DE);
modelEn = new Model(SPEECHTOTEXT_MODEL_EN);
final SidPlay2Section sidplay2Section = configuration.getSidplay2Section();
deepFilterOutputDir = new File(sidplay2Section.getTmpDir(), UUID.randomUUID().toString());
} catch (IOException e) {
// ignored here, since it's checked during request
}
}
@Override
public void destroy() {
executorService.shutdown();
}
@Override
public Map<Class<? extends HttpFilter>, Map<String, String>> getServletFiltersParameterMap() {
Map<Class<? extends HttpFilter>, Map<String, String>> result = new HashMap<>();
Map<String, String> counterBasedRateLimiterFilterParameters = new HashMap<>();
counterBasedRateLimiterFilterParameters.put(FILTER_PARAMETER_MAX_REQUESTS_PER_SERVLET,
String.valueOf(MAX_SPEECH_TO_TEXT));
result.put(CounterBasedRateLimiterFilter.class, counterBasedRateLimiterFilterParameters);
return result;
}
/**
* Speech recognition.
* <p>
* E.g. {@code http://haendel.ddns.net:8080/jsidplay2service/JSIDPlay2REST/speech2text }
*/
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response)
throws IOException {
HttpSession session = request.getSession(true);
session.setMaxInactiveInterval(SPEECHTOTEXT_SESSION_MAX_INACTIVE_INTERVAL);
Object sessionLock = SessionLockManager.getSessionLock(session);
AsyncContext asyncContext = request.startAsync(request, response);
asyncContext.setTimeout(SPEECHTOTEXT_ASYNC_TIMEOUT);
try {
executorService.execute(new HttpAsyncContextRunnable(asyncContext, getServletContext()) {
public void run(HttpServletRequest request, HttpServletResponse response) {
// thread-safety per session
synchronized (sessionLock) {
try {
// first read Blob, then parameters!
AudioBuffer audioBuffer = AudioBuffer.getOrCreateAudioBuffer(session);
audioBuffer.append(request);
final SpeechToTextServletParameters servletParameters = new SpeechToTextServletParameters();
ServletParameterParser parser = new ServletParameterParser(request, response,
servletParameters);
if (servletParameters.getHelp() || parser.hasException()) {
parser.usage();
return;
}
Locale speechToTextLocale = getSpeechToTextLocale(servletParameters);
int samplingRate = servletParameters.getSamplingRate();
String result;
byte[] bytesToRecognize = deepFilter(samplingRate, audioBuffer.getBytesAt(0));
try (Recognizer recognizer = new Recognizer(getModel(speechToTextLocale),
samplingRate)) {
recognizer.acceptWaveForm(bytesToRecognize, bytesToRecognize.length);
result = parseJsonResult(recognizer.getFinalResult());
if (Optional.ofNullable(servletParameters.getCommands())
.orElse(Collections.emptyList()).stream().anyMatch(result::contains)) {
// Any command recognized? End of command assumed!
audioBuffer.clear();
}
}
audioBuffer.sliceEnd(SPEECHTOTEXT_MAX_RECORDING_TIME * samplingRate * Short.BYTES);
if (!isComplete()) {
setOutput(request, response, result);
}
} catch (IllegalArgumentException e) {
error(getServletContext(), e, parentThread);
if (!isComplete()) {
response.setStatus(HttpServletResponse.SC_CONFLICT);
}
} catch (Throwable t) {
error(getServletContext(), t, parentThread);
if (!isComplete()) {
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
setOutput(response, t);
}
}
}
}
private Locale getSpeechToTextLocale(SpeechToTextServletParameters servletParameters) {
return servletParameters.getSpeechToTextLocale() != null ? servletParameters.getSpeechToTextLocale()
: servletParameters.getLocale();
}
private Model getModel(Locale speechToTextLocale) throws IOException {
Model model = speechToTextLocale == Locale.GERMAN ? modelDe : modelEn;
if (model == null) {
throw new IOException(format("Vosk model not available: %s",
speechToTextLocale == Locale.GERMAN ? SPEECHTOTEXT_MODEL_DE : SPEECHTOTEXT_MODEL_EN));
}
return model;
}
private byte[] deepFilter(int samplingRate, byte[] bytesToRecognize)
throws IOException {
if (!SPEECHTOTEXT_DEEPFILTER || samplingRate != 48000) {
return bytesToRecognize;
}
SidPlay2Section sidplay2Section = configuration.getSidplay2Section();
File wavFile = File.createTempFile("speech2text", ".wav", sidplay2Section.getTmpDir());
WAVHeader wavHeader = new WAVHeader(1, samplingRate);
wavHeader.advance(bytesToRecognize.length);
try (InputStream is = new ByteArrayInputStream(bytesToRecognize);
OutputStream os = Files.newOutputStream(wavFile.toPath())) {
os.write(wavHeader.getBytes());
IOUtils.copy(is, os);
}
new ExternalProcess().execute(new DeepFilterParameters(wavFile, deepFilterOutputDir));
File filteredWav = new File(deepFilterOutputDir, wavFile.getName());
ByteArrayOutputStream bos = new ByteArrayOutputStream();
try (InputStream is = Files.newInputStream(filteredWav.toPath())) {
ioWrap(() -> is.read(new byte[WAVHeader.HEADER_LENGTH]));
IOUtils.copy(is, bos);
}
byte[] byteArray = bos.toByteArray();
ioWrap(wavFile::delete);
ioWrap(filteredWav::delete);
return Arrays.copyOf(byteArray, byteArray.length);
}
private String parseJsonResult(String jsonResult) throws JsonProcessingException {
JsonNode root = new ObjectMapper().readTree(jsonResult);
JsonNode textNode = root.get("text");
return textNode != null ? textNode.asText() : "";
}
});
} catch (RejectedExecutionException e) {
error(getServletContext(), e);
response.sendError(SC_TOO_MANY_REQUESTS, "Too Many Requests");
}
}
}