XuggleVideoDriver.java
package sidplay.audio.xuggle;
import com.xuggle.xuggler.*;
import com.xuggle.xuggler.ICodec.ID;
import com.xuggle.xuggler.video.ConverterFactory;
import com.xuggle.xuggler.video.IConverter;
import libsidplay.common.CPUClock;
import libsidplay.common.Event.Phase;
import libsidplay.common.EventScheduler;
import libsidplay.common.SamplingRate;
import libsidplay.components.mos656x.MOS6567;
import libsidplay.components.mos656x.VIC;
import libsidplay.config.IAudioSection;
import sidplay.audio.AudioConfig;
import sidplay.audio.AudioDriver;
import sidplay.audio.VideoDriver;
import sidplay.audio.exceptions.IniConfigException;
import javax.sound.sampled.LineUnavailableException;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferInt;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
import static com.xuggle.xuggler.IAudioSamples.Format.FMT_S16;
import static com.xuggle.xuggler.IContainer.Type.WRITE;
import static com.xuggle.xuggler.IPixelFormat.Type.YUV420P;
import static com.xuggle.xuggler.IStreamCoder.Flags.FLAG_QSCALE;
import static java.awt.image.BufferedImage.TYPE_INT_ARGB;
import static java.lang.Short.BYTES;
import static java.nio.ByteOrder.LITTLE_ENDIAN;
import static libsidplay.common.CPUClock.PAL;
import static libsidplay.components.mos656x.MOS6569.BORDER_HEIGHT;
import static libsidplay.components.mos656x.VIC.MAX_HEIGHT;
import static libsidplay.components.mos656x.VIC.MAX_WIDTH;
import static libsidutils.C64FontUtils.TRUE_TYPE_FONT_BIG;
import static libsidutils.C64FontUtils.petsciiToFont;
/**
* Create video.<BR>
*
* <pre>
* Video possibilities (file, RTMP, and HLS):
* /home/ken/.jsidplay2/jsidplay2.flv
* rtmp://localhost/live/test
* <a href="http://localhost:90/hls/test.m3u8">test.m3u8</a> (provided by NGINX RTMP as HLS)
* </pre>
*
* <B>The Kush Gauge:</B> To find a decent bitrate simply multiply the target
* pixel count by the frame rate; then multiply the result by a factor of 1, 2 or 4, depending on the amount of motion
* in the video; and then multiply that result by 0.07 to get the bit rate in bps. <br>
*
* <pre>
* Video Coder Bit Rate:
* PAL low motion: 192 * 312 * 50,1246 * 1 * 0,07 = 210.186
* PAL high motion: 192 * 312 * 50,1246 * 4 * 0,07 = 840.746
* NTSC low motion: 192 * 312 * 59,83 * 1 * 0,07 = 250.884
* NTSC high motion: 192 * 312 * 59,83 * 4 * 0,07 = 1.003.536
* </pre>
*
* @author ken
*
*/
public abstract class XuggleVideoDriver extends XuggleBase implements AudioDriver, VideoDriver {
private static final int STATUS_TEXT_Y = 10;
private EventScheduler context;
private IContainer container;
private IStreamCoder videoCoder, audioCoder;
private IConverter converter;
private BufferedImage vicImage;
private IntBuffer pictureBuffer;
private int[] statusTextPixels;
private int statusTextOffset, statusTextX, statusTextOverflow;
private long frameNo, framesPerKeyFrames, firstAudioTimeStamp, firstVideoTimeStamp;
private double ticksPerMicrosecond;
private int audioDelayInMs;
private ByteBuffer sampleBuffer;
private double screenRefresh;
private int borderHeight;
private boolean videoTimingConfigured;
@Override
public void setVideoTiming(double screenRefresh, int borderHeight) {
this.screenRefresh = screenRefresh;
this.borderHeight = borderHeight;
videoTimingConfigured = true;
}
@Override
public void open(IAudioSection audioSection, String recordingFilename, CPUClock cpuClock, EventScheduler context)
throws IOException, LineUnavailableException, InterruptedException {
this.context = context;
AudioConfig cfg = new AudioConfig(audioSection);
String url = getUrl(audioSection, recordingFilename);
if (url == null) {
throw new FileNotFoundException("url is missing, please set option --vcStreamingUrl");
}
if (!getSupportedSamplingRates().contains(audioSection.getSamplingRate())) {
throw new IniConfigException("Sampling rate is not supported by encoder, switch to default",
() -> audioSection.setSamplingRate(getDefaultSamplingRate()));
}
container = IContainer.make();
IContainerFormat containerFormat = IContainerFormat.make();
containerFormat.setOutputFormat(getOutputFormatName(), url, null);
container.setInputBufferLength(0);
throwExceptionOnError(container.open(url, WRITE, containerFormat), "Could not open: '" + url + "'");
if (!videoTimingConfigured) {
screenRefresh = cpuClock.getScreenRefresh();
borderHeight = cpuClock == PAL ? BORDER_HEIGHT : MOS6567.BORDER_HEIGHT;
}
videoTimingConfigured = false;
videoCoder = createVideoCoder(audioSection);
throwExceptionOnError(videoCoder.open(null, null));
audioCoder = createAudioCoder(audioSection, cfg);
throwExceptionOnError(audioCoder.open(null, null));
container.writeHeader();
vicImage = new BufferedImage(MAX_WIDTH, MAX_HEIGHT, TYPE_INT_ARGB);
pictureBuffer = IntBuffer.wrap(((DataBufferInt) vicImage.getRaster().getDataBuffer()).getData());
converter = ConverterFactory.createConverter(vicImage, YUV420P);
setStatusText("Recorded by JSIDPlay2!");
frameNo = 0;
firstAudioTimeStamp = 0;
firstVideoTimeStamp = 0;
statusTextOffset = MAX_WIDTH * (borderHeight + STATUS_TEXT_Y);
statusTextX = 0;
statusTextOverflow = 0;
audioDelayInMs = audioSection.getVideoCoderAudioDelay();
ticksPerMicrosecond = cpuClock.getCpuFrequency() / 1000000;
framesPerKeyFrames = (int) screenRefresh;
sampleBuffer = ByteBuffer.allocate(cfg.getChunkFrames() * BYTES * cfg.getChannels()).order(LITTLE_ENDIAN);
}
@Override
public void write() throws InterruptedException {
long timeStamp = getAudioTimeStamp();
int numSamples = sampleBuffer.position() >> 2;
IAudioSamples audioSamples = IAudioSamples.make(numSamples, audioCoder.getChannels(), FMT_S16);
audioSamples.getData().put(sampleBuffer.array(), 0, 0, sampleBuffer.position());
audioSamples.setComplete(true, numSamples, audioCoder.getSampleRate(), audioCoder.getChannels(), FMT_S16,
timeStamp);
int samplesConsumed = 0;
IPacket packet = IPacket.make();
while (samplesConsumed < audioSamples.getNumSamples()) {
samplesConsumed += throwExceptionOnError(audioCoder.encodeAudio(packet, audioSamples, samplesConsumed));
if (packet.isComplete()) {
container.writePacket(packet);
}
}
audioSamples.delete();
packet.delete();
}
@Override
public void accept(VIC vic) {
long timeStamp = getVideoTimeStamp();
ByteBuffer pixels = vic.getPalEmulation().getPixels();
ByteBuffer pixelsCopy = pixels.asReadOnlyBuffer().order(pixels.order());
pixelsCopy.rewind();
IntBuffer pixelsAsIntBuffer = pixelsCopy.asIntBuffer();
pixelsAsIntBuffer.limit(statusTextOffset);
pictureBuffer.put(pixelsAsIntBuffer).put(statusTextPixels);
pictureBuffer.clear();
IVideoPicture videoPicture = converter.toPicture(vicImage, timeStamp);
videoPicture.setKeyFrame((frameNo++ % framesPerKeyFrames) == 0);
IPacket packet = IPacket.make();
throwExceptionOnError(videoCoder.encodeVideo(packet, videoPicture, 0));
if (packet.isComplete()) {
container.writePacket(packet);
}
videoPicture.delete();
packet.delete();
}
@Override
public void close() {
if (container != null) {
if (container.isOpened()) {
container.writeTrailer();
}
if (audioCoder != null) {
if (audioCoder.isOpen()) {
audioCoder.close();
}
audioCoder = null;
}
if (videoCoder != null) {
if (videoCoder.isOpen()) {
videoCoder.close();
}
videoCoder = null;
}
if (container.isOpened()) {
container.close();
}
container = null;
}
}
@Override
public ByteBuffer buffer() {
return sampleBuffer;
}
@Override
public boolean isRecording() {
return true;
}
public void setStatusText(String text) {
String fontText = petsciiToFont(text.toUpperCase(Locale.ENGLISH), TRUE_TYPE_FONT_BIG);
BufferedImage statusImage = new BufferedImage(MAX_WIDTH, c64Font.getSize(), TYPE_INT_ARGB);
Graphics2D graphics = statusImage.createGraphics();
graphics.setFont(c64Font);
graphics.drawString(fontText, -statusTextX, graphics.getFontMetrics(c64Font).getAscent());
graphics.dispose();
statusTextOverflow = Math.max(0, c64Font.getSize() * text.length() - statusTextX - MAX_WIDTH);
statusTextPixels = ((DataBufferInt) statusImage.getRaster().getDataBuffer()).getData();
}
public int getStatusTextOverflow() {
return statusTextOverflow;
}
public int getStatusTextX() {
return statusTextX;
}
public void setStatusTextX(int statusTextX) {
this.statusTextX = statusTextX;
}
private IStreamCoder createVideoCoder(IAudioSection audioSection) {
IStreamCoder videoCoder = container.addNewStream(getVideoCodec()).getStreamCoder();
videoCoder.setNumPicturesInGroupOfPictures(audioSection.getVideoCoderNumPicturesInGroupOfPictures());
videoCoder.setBitRate(audioSection.getVideoCoderBitRate());
videoCoder.setBitRateTolerance(audioSection.getVideoCoderBitRateTolerance());
videoCoder.setTimeBase(IRational.make(1 / screenRefresh));
videoCoder.setPixelType(YUV420P);
videoCoder.setHeight(MAX_HEIGHT);
videoCoder.setWidth(MAX_WIDTH);
videoCoder.setFlag(FLAG_QSCALE, true);
videoCoder.setGlobalQuality(audioSection.getVideoCoderGlobalQuality());
configurePreset(videoCoder, audioSection.getVideoCoderPreset().getPresetName());
return videoCoder;
}
private void configurePreset(IConfigurable configurable, String presetName) {
try (InputStream is = XuggleVideoDriver.class.getResourceAsStream(presetName)) {
Properties props = new Properties();
props.load(is);
Configuration.configure(props, configurable);
} catch (IOException | NullPointerException e) {
throw new RuntimeException("You need preset " + presetName + " in your classpath.");
}
}
private IStreamCoder createAudioCoder(IAudioSection audioSection, AudioConfig cfg) {
IStreamCoder audioCoder = container.addNewStream(getAudioCodec()).getStreamCoder();
audioCoder.setChannels(cfg.getChannels());
audioCoder.setSampleFormat(FMT_S16);
audioCoder.setBitRate(audioSection.getAudioCoderBitRate());
audioCoder.setBitRateTolerance(audioSection.getAudioCoderBitRateTolerance());
audioCoder.setSampleRate(cfg.getFrameRate());
return audioCoder;
}
private long getAudioTimeStamp() {
long now = context.getTime(Phase.PHI2);
if (firstAudioTimeStamp == 0) {
firstAudioTimeStamp = now;
}
return (long) ((now - firstAudioTimeStamp) / ticksPerMicrosecond) + (audioDelayInMs * 1000L);
}
private long getVideoTimeStamp() {
long now = context.getTime(Phase.PHI2);
if (firstVideoTimeStamp == 0) {
firstVideoTimeStamp = now;
}
return (long) ((now - firstVideoTimeStamp) / ticksPerMicrosecond);
}
protected abstract String getOutputFormatName();
protected abstract List<SamplingRate> getSupportedSamplingRates();
protected abstract SamplingRate getDefaultSamplingRate();
protected abstract ID getVideoCodec();
protected abstract ID getAudioCodec();
protected abstract String getUrl(IAudioSection audioSection, String recordingFilename);
}