SpeechToTextServlet.java

package server.restful.servlets;

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.AsyncContext;
import jakarta.servlet.annotation.HttpConstraint;
import jakarta.servlet.annotation.ServletSecurity;
import jakarta.servlet.annotation.WebServlet;
import jakarta.servlet.http.HttpFilter;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.servlet.http.HttpSession;
import libsidutils.IOUtils;
import org.vosk.LibVosk;
import org.vosk.LogLevel;
import org.vosk.Model;
import org.vosk.Recognizer;
import server.restful.common.JSIDPlay2Servlet;
import server.restful.common.SessionLockManager;
import server.restful.common.async.DefaultThreadFactory;
import server.restful.common.async.HttpAsyncContextRunnable;
import server.restful.common.converter.LocaleConverter;
import server.restful.common.filters.CounterBasedRateLimiterFilter;
import server.restful.common.parameter.RequestResponseBodyParameter;
import server.restful.common.parameter.RequestResponseBodyParameter.Blob;
import server.restful.common.parameter.ServletParameterParser;
import server.restful.common.parameter.UsageParameter;
import server.restful.common.processor.ExternalProcess;
import server.restful.common.speech2text.AudioBuffer;
import server.restful.common.speech2text.DeepFilterParameters;
import sidplay.audio.wav.WAVHeader;
import ui.entities.config.SidPlay2Section;

import java.io.*;
import java.nio.file.Files;
import java.util.*;
import java.util.concurrent.*;

import static java.lang.Integer.MIN_VALUE;
import static java.lang.String.format;
import static libsidutils.IOUtils.ioWrap;
import static org.apache.http.HttpStatus.SC_TOO_MANY_REQUESTS;
import static server.restful.JSIDPlay2Server.*;
import static server.restful.common.IServletSystemProperties.SpeechToTextServlet.*;
import static server.restful.common.ServletUtil.error;
import static server.restful.common.filters.CounterBasedRateLimiterFilter.FILTER_PARAMETER_MAX_REQUESTS_PER_SERVLET;

@WebServlet(name = "SpeechToTextServlet", displayName = "SpeechToTextServlet", asyncSupported = true, urlPatterns = CONTEXT_ROOT_SERVLET
        + "/speech2text", description = "Speech recognition")
@ServletSecurity(value = @HttpConstraint(rolesAllowed = {ROLE_USER, ROLE_ADMIN}))
public class SpeechToTextServlet extends JSIDPlay2Servlet {

    @SuppressWarnings("unused")
    @Parameters(resourceBundle = "server.restful.servlets.SpeechToTextServletParameters")
    public static class SpeechToTextServletParameters extends UsageParameter
            implements RequestResponseBodyParameter<Blob, String> {

        private Locale locale = Locale.ROOT;

        public Locale getLocale() {
            return locale;
        }

        @Parameter(names = "--locale", descriptionKey = "LOCALE", converter = LocaleConverter.class, order = MIN_VALUE
                + 1)
        public void setLocale(Locale locale) {
            this.locale = locale;
        }

        private Locale speechToTextLocale;

        public Locale getSpeechToTextLocale() {
            return speechToTextLocale;
        }

        @Parameter(names = "--speechToTextLocale", descriptionKey = "SPEECH_TO_TEXT_LOCALE", converter = LocaleConverter.class, order = MIN_VALUE
                + 2)
        public void setSpeechToTextLocale(Locale speechToTextLocale) {
            this.speechToTextLocale = speechToTextLocale;
        }

        private int samplingRate = 48000;

        public int getSamplingRate() {
            return samplingRate;
        }

        @Parameter(names = "--samplingRate", descriptionKey = "FREQUENCY", order = MIN_VALUE + 3)
        public void setSamplingRate(int samplingRate) {
            this.samplingRate = samplingRate;
        }

        private List<String> commands;

        public List<String> getCommands() {
            return commands;
        }

        @Parameter(names = "--commands", descriptionKey = "COMMANDS", order = MIN_VALUE + 4)
        public void setCommands(List<String> commands) {
            this.commands = commands;
        }

        @Override
        public Blob getRequestBodyExample() {
            return new Blob();
        }

        @Override
        public String getResponseBodyExample() {
            return "Hello World";
        }
    }

    private ExecutorService executorService;
    private Model modelDe, modelEn;
    private File deepFilterOutputDir;

    @Override
    public void init() {
        executorService = new ThreadPoolExecutor(MAX_SPEECHTOTEXT_IN_PARALLEL_CORE, MAX_SPEECHTOTEXT_IN_PARALLEL,
                SPEECHTOTEXT_KEEPALIVE_TIME, TimeUnit.SECONDS, new ArrayBlockingQueue<>(SPEECHTOTEXT_QUEUE_SIZE),
                new DefaultThreadFactory("/speechtotext"), new ThreadPoolExecutor.AbortPolicy());
        try {
            LibVosk.setLogLevel(LogLevel.WARNINGS);
            modelDe = new Model(SPEECHTOTEXT_MODEL_DE);
            modelEn = new Model(SPEECHTOTEXT_MODEL_EN);
            final SidPlay2Section sidplay2Section = configuration.getSidplay2Section();
            deepFilterOutputDir = new File(sidplay2Section.getTmpDir(), UUID.randomUUID().toString());
        } catch (IOException e) {
            // ignored here, since it's checked during request
        }
    }

    @Override
    public void destroy() {
        executorService.shutdown();
    }

    @Override
    public Map<Class<? extends HttpFilter>, Map<String, String>> getServletFiltersParameterMap() {
        Map<Class<? extends HttpFilter>, Map<String, String>> result = new HashMap<>();

        Map<String, String> counterBasedRateLimiterFilterParameters = new HashMap<>();
        counterBasedRateLimiterFilterParameters.put(FILTER_PARAMETER_MAX_REQUESTS_PER_SERVLET,
                String.valueOf(MAX_SPEECH_TO_TEXT));
        result.put(CounterBasedRateLimiterFilter.class, counterBasedRateLimiterFilterParameters);

        return result;
    }

    /**
     * Speech recognition.
     * <p>
     * E.g. {@code http://haendel.ddns.net:8080/jsidplay2service/JSIDPlay2REST/speech2text }
     */
    @Override
    protected void doPost(HttpServletRequest request, HttpServletResponse response)
            throws IOException {
        HttpSession session = request.getSession(true);
        session.setMaxInactiveInterval(SPEECHTOTEXT_SESSION_MAX_INACTIVE_INTERVAL);
        Object sessionLock = SessionLockManager.getSessionLock(session);

        AsyncContext asyncContext = request.startAsync(request, response);
        asyncContext.setTimeout(SPEECHTOTEXT_ASYNC_TIMEOUT);

        try {
            executorService.execute(new HttpAsyncContextRunnable(asyncContext, getServletContext()) {
                public void run(HttpServletRequest request, HttpServletResponse response) {
                    // thread-safety per session
                    synchronized (sessionLock) {
                        try {
                            // first read Blob, then parameters!
                            AudioBuffer audioBuffer = AudioBuffer.getOrCreateAudioBuffer(session);
                            audioBuffer.append(request);

                            final SpeechToTextServletParameters servletParameters = new SpeechToTextServletParameters();

                            ServletParameterParser parser = new ServletParameterParser(request, response,
                                    servletParameters);

                            if (servletParameters.getHelp() || parser.hasException()) {
                                parser.usage();
                                return;
                            }
                            Locale speechToTextLocale = getSpeechToTextLocale(servletParameters);
                            int samplingRate = servletParameters.getSamplingRate();

                            String result;
                            byte[] bytesToRecognize = deepFilter(samplingRate, audioBuffer.getBytesAt(0));
                            try (Recognizer recognizer = new Recognizer(getModel(speechToTextLocale),
                                    samplingRate)) {
                                recognizer.acceptWaveForm(bytesToRecognize, bytesToRecognize.length);
                                result = parseJsonResult(recognizer.getFinalResult());
                                if (Optional.ofNullable(servletParameters.getCommands())
                                        .orElse(Collections.emptyList()).stream().anyMatch(result::contains)) {
                                    // Any command recognized? End of command assumed!
                                    audioBuffer.clear();
                                }
                            }
                            audioBuffer.sliceEnd(SPEECHTOTEXT_MAX_RECORDING_TIME * samplingRate * Short.BYTES);

                            if (!isComplete()) {
                                setOutput(request, response, result);
                            }

                        } catch (IllegalArgumentException e) {
                            error(getServletContext(), e, parentThread);
                            if (!isComplete()) {
                                response.setStatus(HttpServletResponse.SC_CONFLICT);
                            }
                        } catch (Throwable t) {
                            error(getServletContext(), t, parentThread);
                            if (!isComplete()) {
                                response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
                                setOutput(response, t);
                            }
                        }
                    }
                }

                private Locale getSpeechToTextLocale(SpeechToTextServletParameters servletParameters) {
                    return servletParameters.getSpeechToTextLocale() != null ? servletParameters.getSpeechToTextLocale()
                            : servletParameters.getLocale();
                }

                private Model getModel(Locale speechToTextLocale) throws IOException {
                    Model model = speechToTextLocale == Locale.GERMAN ? modelDe : modelEn;

                    if (model == null) {
                        throw new IOException(format("Vosk model not available: %s",
                                speechToTextLocale == Locale.GERMAN ? SPEECHTOTEXT_MODEL_DE : SPEECHTOTEXT_MODEL_EN));
                    }
                    return model;
                }

                private byte[] deepFilter(int samplingRate, byte[] bytesToRecognize)
                        throws IOException {
                    if (!SPEECHTOTEXT_DEEPFILTER || samplingRate != 48000) {
                        return bytesToRecognize;
                    }
                    SidPlay2Section sidplay2Section = configuration.getSidplay2Section();
                    File wavFile = File.createTempFile("speech2text", ".wav", sidplay2Section.getTmpDir());
                    WAVHeader wavHeader = new WAVHeader(1, samplingRate);
                    wavHeader.advance(bytesToRecognize.length);
                    try (InputStream is = new ByteArrayInputStream(bytesToRecognize);
                         OutputStream os = Files.newOutputStream(wavFile.toPath())) {
                        os.write(wavHeader.getBytes());
                        IOUtils.copy(is, os);
                    }
                    new ExternalProcess().execute(new DeepFilterParameters(wavFile, deepFilterOutputDir));
                    File filteredWav = new File(deepFilterOutputDir, wavFile.getName());
                    ByteArrayOutputStream bos = new ByteArrayOutputStream();
                    try (InputStream is = Files.newInputStream(filteredWav.toPath())) {
                        ioWrap(() -> is.read(new byte[WAVHeader.HEADER_LENGTH]));
                        IOUtils.copy(is, bos);
                    }
                    byte[] byteArray = bos.toByteArray();
                    ioWrap(wavFile::delete);
                    ioWrap(filteredWav::delete);
                    return Arrays.copyOf(byteArray, byteArray.length);
                }

                private String parseJsonResult(String jsonResult) throws JsonProcessingException {
                    JsonNode root = new ObjectMapper().readTree(jsonResult);
                    JsonNode textNode = root.get("text");
                    return textNode != null ? textNode.asText() : "";
                }

            });
        } catch (RejectedExecutionException e) {
            error(getServletContext(), e);
            response.sendError(SC_TOO_MANY_REQUESTS, "Too Many Requests");
        }

    }

}