Hi again Kas and all...
Here's a patch for master, which also applies to redacted in case that
helps. I found out that you don't need parantheses when defining
aliases to other functions, as in boot.scm there is the reason for
that other oddity: (define time flxtime). I changed the docs to
reflect that, you can (define time process-time) and (define flxtime
process-time) or only the one you used, if one... I always used
(time). I tested this with a script that draws *many* cubes in
immediate mode with the number being based on several harmonic values,
and it sometimes takes a second to unfreeze because there is so much
overdraw and I'm using (blend-mode 'one 'one). It's pretty smooth...
and to get mplayer to work, I just had to use tif and then convert all
to tga which didn't cause it to complain about invalid colorspace like
png did. So that's cool.
There seems to be little or no use for m_BufferTime in AudioCollector,
and it is the only obvious thing depending on m_Samplerate. However,
over in the big mess, I also had to use m_Samplerate for the bandwidth
calculations to build the frequency scaling matrix, (in case that
sounds smart, I only translated it from matlab code which would have
taken me months to write) so it made sense to change m_Samplerate to
the sample rate of the wav being worked on. Then I had to change it
back to JackClient's sample rate when the wav finished, so that's why
there are some includes moved around and a persistently visible Jack
pointer. It's not exactly necessary in master but I wanted to avoid
possible weirdness if it had a purpose I missed.
This provides 4 new functions and changes one. (process) now takes 2
arguments, the string location of the wav and a number for the
framerate, which can be inexact. (process-time) returns the inexact
number of seconds into the wav being processed, and (process-delta)
returns a constant: 1/framerate. (stop-process) closes the wav and
returns to normal early if you need to. (processing) returns true if a
wav is open and in use.
You can set any buffer size you like when calling (start-audio), and
it will only affect the FFT precision and size of (ga)-- nothing needs
to be done to make the wav run through correctly.
If I goofed, it's because after it worked I had to make it work again
by pasting into an editor that didn't fix a bunch of whitespace and
enlarge the commit, so good luck with that, and thanks for the
feedback.
--
Rob
diff --git a/modules/fluxus-audio/src/AudioCollector.cpp b/modules/fluxus-audio/src/AudioCollector.cpp
index 1077bbd..9e48433 100644
--- a/modules/fluxus-audio/src/AudioCollector.cpp
+++ b/modules/fluxus-audio/src/AudioCollector.cpp
@@ -17,9 +17,7 @@
#include <cstring>
#include <limits.h>
#include <iostream>
-#include <sndfile.h>
#include "AudioCollector.h"
-#include "JackClient.h"
static const int MAX_FFT_LENGTH = 4096;
@@ -110,7 +108,7 @@ m_NumBars(16)
m_Mutex = new pthread_mutex_t;
pthread_mutex_init(m_Mutex,NULL);
- JackClient *Jack = JackClient::Get();
+ Jack = JackClient::Get();
Jack->SetCallback(AudioCallback,(void*)this);
Jack->Attach(portname);
if (Jack->IsAttached())
@@ -144,19 +142,49 @@ float *AudioCollector::GetFFT()
{
if (m_Processing)
{
- if (m_ProcessPos+m_BufferLength<m_ProcessLength)
+ float mix;
+ // grab next chunk of wav data
+ sf_count_t seekpos = sf_seek(m_ProcWavFile, (sf_count_t)floor(((double)m_ProcWavInfo.samplerate / m_ProcessFPS) * (double)m_ProcessPos), SEEK_SET);
+ if (seekpos == -1)
{
- m_FFT.Impulse2Freq(m_ProcessBuffer+m_ProcessPos,m_FFTBuffer);
- memcpy((void*)m_AudioBuffer,(void*)(m_ProcessBuffer+m_ProcessPos),m_BufferLength*sizeof(float));
- m_ProcessPos+=m_BufferLength;
+ //shouldn't happen
+ cerr << "WAV seek failed, already finished audio file? closing." << endl;
+ EndProcessing();
+ return m_FFTOutput; //no update this frame
}
- else
+ sf_count_t frames = sf_readf_float(m_ProcWavFile, m_ProcessBuffer, m_BufferLength);
+ for (unsigned int i = 0; i < m_BufferLength; i++)
{
- cerr<<"Finished processing audio file..."<<endl;
+ if ((long long)i < frames)
+ {
+ if (m_ProcWavInfo.channels > 1)
+ {
+ //average them together linearly
+ mix = 0.0f;
+ for (unsigned int b = 0; b < m_ProcWavInfo.channels; b++)
+ {
+ mix += m_ProcessBuffer[(i * m_ProcWavInfo.channels) + b];
+ m_AudioBuffer[i] = mix / (float)m_ProcWavInfo.channels;
+ }
+ } else {
+ m_AudioBuffer[i] = m_ProcessBuffer[i];
+ }
+ } else {
+ //got to the end during this frame, zero pad
+ m_AudioBuffer[i] = 0.0;
+ }
+ }
+
+ m_FFT.Impulse2Freq(m_AudioBuffer,m_FFTBuffer);
+
+ if (frames == (long long)m_BufferLength)
+ {
+ //got right size, not done yet
+ m_ProcessPos++;
+ } else {
+ cerr << "Finished processing audio file..." << endl;
// finished, so clean up...
- delete[] m_ProcessBuffer;
- m_ProcessPos=0;
- m_Processing=false;
+ EndProcessing();
}
}
else
@@ -199,46 +227,47 @@ float *AudioCollector::GetFFT()
}
-void AudioCollector::Process(const string &filename)
+void AudioCollector::StartProcessing(const string &filename, double fps)
{
if (m_Processing) return;
- SF_INFO info;
- info.format=0;
+ if (!pthread_mutex_lock(m_Mutex)) {
+ m_ProcessFPS = fps;
+ m_ProcessDelta = 1.0 / fps;
+ m_ProcWavInfo.format = 0;
- SNDFILE* file = sf_open (filename.c_str(), SFM_READ, &info) ;
- if (!file)
- {
- cerr<<"Error opening ["<<filename<<"] : "<<sf_strerror(file)<<endl;
- return;
- }
+ m_ProcWavFile = sf_open(filename.c_str(), SFM_READ, &m_ProcWavInfo) ;
+ if (!m_ProcWavFile) {
+ cerr << "Error opening [" << filename << "] : " << sf_strerror(m_ProcWavFile) << endl;
+ return;
+ }
+ cerr << "Successfully opened the file " << filename << " for reading" << endl;
- m_ProcessBuffer = new float[info.frames];
- memset((void*)m_ProcessBuffer,0,info.frames*sizeof(float));
- m_ProcessLength=info.frames;
+ m_ProcessBuffer = new float[m_BufferLength * m_ProcWavInfo.channels];
+ for (unsigned int i = 0; i < (m_BufferLength * m_ProcWavInfo.channels); i++)
+ m_ProcessBuffer[i] = 0.0f;
+ cerr << "Successfully allocated process buffer" << endl;
- // mix down to mono if need be
- if (info.channels>1)
- {
- float *Buffer = new float[info.frames*info.channels];
- sf_readf_float(file,Buffer,info.frames*info.channels);
- int from=0;
- for (int n=0; n<info.frames; n++)
- {
- for (int c=0; c<info.channels; c++)
- {
- m_ProcessBuffer[n]=(m_ProcessBuffer[n]+Buffer[from++])/2.0f;
- }
- }
- }
- else
- {
- sf_readf_float(file, m_ProcessBuffer, info.frames);
+ m_ProcessLength = m_ProcWavInfo.frames;
+ m_Samplerate = m_ProcWavInfo.samplerate;
+ m_BufferTime = m_BufferLength/(float)m_Samplerate;
+ m_ProcessPos = 0;
+ pthread_mutex_unlock(m_Mutex);
+ m_Processing = true;
}
- sf_close(file);
+}
- m_Processing=true;
- m_ProcessPos=0;
+void AudioCollector::EndProcessing(void)
+{
+ if (!m_Processing) return;
+ sf_close(m_ProcWavFile);
+ delete[] m_ProcessBuffer;
+ m_ProcessBuffer = NULL;
+ m_Samplerate = Jack->JackClient::GetSamplerate();
+ m_BufferTime = m_BufferLength/(float)m_Samplerate;
+ m_ProcessPos = 0;
+ m_ProcessDelta = 0.0;
+ m_Processing = false;
}
void AudioCollector::AudioCallback_i(unsigned int Size)
diff --git a/modules/fluxus-audio/src/AudioCollector.h b/modules/fluxus-audio/src/AudioCollector.h
index 52b9f3d..a35eb13 100644
--- a/modules/fluxus-audio/src/AudioCollector.h
+++ b/modules/fluxus-audio/src/AudioCollector.h
@@ -18,6 +18,9 @@
#include <fftw3.h>
#include <pthread.h>
#include <string>
+#include <cmath>
+#include <sndfile.h>
+#include "JackClient.h"
#ifndef AUDIO_COLLECTOR
#define AUDIO_COLLECTOR
@@ -58,7 +61,10 @@ public:
void SetGain(float s) { m_Gain=s; }
float GetGain() { return m_Gain; }
void SetSmoothingBias(float s) { if ((s < 1) && (s >= 0)) m_SmoothingBias = s; }
- void Process(const string &filename);
+ void StartProcessing(const string &filename, double fps);
+ void EndProcessing(void);
+ double ProcessTimeStamp(void) { if (!m_Processing) return 0.0f; return (double)m_ProcessPos / m_ProcessFPS; }
+ double ProcessDelta(void) { if (!m_Processing) return 0.0f; return m_ProcessDelta; }
bool IsProcessing() { return m_Processing; }
float BufferTime() { return m_BufferTime; }
@@ -72,6 +78,7 @@ public:
}
unsigned GetNumBars(void) { return m_NumBars; }
+ JackClient *Jack;
private:
@@ -98,7 +105,11 @@ private:
short *m_OSSBuffer;
float m_OneOverSHRT_MAX;
bool m_Processing;
+ SF_INFO m_ProcWavInfo;
+ SNDFILE *m_ProcWavFile;
float *m_ProcessBuffer;
+ double m_ProcessDelta;
+ double m_ProcessFPS;
unsigned int m_ProcessPos;
unsigned int m_ProcessLength;
unsigned int m_NumBars;
diff --git a/modules/fluxus-audio/src/FluxusAudio.cpp b/modules/fluxus-audio/src/FluxusAudio.cpp
index 1ed83e2..6e428b1 100644
--- a/modules/fluxus-audio/src/FluxusAudio.cpp
+++ b/modules/fluxus-audio/src/FluxusAudio.cpp
@@ -340,58 +340,133 @@ Scheme_Object *gain(int argc, Scheme_Object **argv)
return scheme_void;
}
+Scheme_Object *process(int argc, Scheme_Object **argv)
+{
+ MZ_GC_DECL_REG(1);
+ MZ_GC_VAR_IN_REG(0, argv);
+ MZ_GC_REG();
+ if (!SCHEME_CHAR_STRINGP(argv[0])) scheme_wrong_type("process", "string", 0, argc, argv);
+ if (!SCHEME_NUMBERP(argv[1])) scheme_wrong_type("process", "real number", 1, argc, argv);
+ char *wavname = scheme_utf8_encode_to_buffer(SCHEME_CHAR_STR_VAL(argv[0]), SCHEME_CHAR_STRLEN_VAL(argv[0]), NULL, 0);
+ double fps = scheme_real_to_double(argv[1]);
+ if (Audio != NULL) {
+ Audio->StartProcessing(wavname, fps);
+ }
+ MZ_GC_UNREG();
+ return scheme_void;
+}
+
// StartFunctionDoc-en
-// process wavfile-string
+// stop-process
// Returns: void
// Description:
-// This command temporarally disables the realtime reading of the input audio stream and reads a
-// wav file instead. For use with the framedump command to process audio offline to make music
-// videos. The advantage of this is that it locks the framerate so the right amount of audio gets
-// read for each frame - making syncing of the frames and audio files possible.
+// When processing a wav file, immediately closes the file and returns to
+// getting audio from jack. Probably only useful in the REPL and during testing
// Example:
-// (process "somemusic.wav") ; read a precorded audio file
+// (process "some.wav" 30)
+// ; and before we finished reading
+// (stop-process)
// EndFunctionDoc
-// StartFunctionDoc-pt
-// process wavfile-string
-// Retorna: void
-// Descrição:
-// Este comando desativa temporariamente a leitura em tempo real da
-// entrada da pista de áudio e lê um arquivo wav ao invés. Para usar
-// com o comando framedump para processar audio offline para fazer
-// videos musicais. A vantagem disto é que ele trava a taxa de quadros
-// então a quantidade certa de áudio é lida para cada quadro - fazendo
-// com que a sincrônia entre quadros e audio seja possível.
-// Exemplo:
-// (process "somemusic.wav") ; read a precorded audio file
+Scheme_Object *stop_process(int argc, Scheme_Object **argv)
+{
+ MZ_GC_DECL_REG(1);
+ MZ_GC_VAR_IN_REG(0, argv);
+ MZ_GC_REG();
+ if (Audio != NULL) {
+ Audio->EndProcessing();
+ }
+ MZ_GC_UNREG();
+ return scheme_void;
+}
+
+// StartFunctionDoc-en
+// process-time
+// Returns: number
+// Description:
+// Call this instead of (time) to cause your animations to be based on
+// the speed of the WAV playback when using (process), instead of the
+// wall time. You can quickly fix your scripts to use this by adding at
+// the top (define time process-time) after (clear) but before calling
+// (or defining?) functions that call (time).
+// Example:
+// (process "some.wav" 30)
+// (every-frame
+// (with-state
+// (rotate (vector (* 0.1 (process-time)) 0 0))
+// (draw-cube)))
// EndFunctionDoc
-// StartFunctionDoc-fr
-// process wavfile-string
-// Retour: void
+Scheme_Object *process_time(int argc, Scheme_Object **argv)
+{
+ Scheme_Object *ret = NULL;
+ MZ_GC_DECL_REG(1);
+ MZ_GC_VAR_IN_REG(0, ret);
+ MZ_GC_REG();
+ if (Audio != NULL) {
+ ret = scheme_make_double(Audio->ProcessTimeStamp());
+ }
+ MZ_GC_UNREG();
+ return ret;
+}
+
+// StartFunctionDoc-en
+// process-delta
+// Returns: number
// Description:
-// Cette commande désactive temporairement la lecture temps-réel du flux audio d'entrée et
-// lis un fichier wav à la place. Peut être utilisé avec la commande framedump pour traiter l'audio
-// séparément pour la création de vidéos avec musique. L'avantage est que le taux d'image est fixe,
-// ainsi la quantité audio lut pour chaque image est correcte - ceci rendant la synchronisation
-// entre image et audio possible.
-// Exemple:
-// (process "somemusic.wav") ; lecture d'un fichier audio pré-enregistré
+// Call this instead of (delta) to cause your animations to be based on
+// the speed of the WAV playback when using (process), instead of the
+// wall time. You can quickly fix your scripts to use this by adding at
+// the top (define delta process-delta) after (clear) but before calling
+// (or defining?) functions that call (delta). Like (delta), this returns the
+// amount of time in seconds that passed since the last frame. However, unlike (delta)
+// this is always 1/FPS because of how (process) works, so this is mainly for convenience.
+// Example:
+// (define c (build-cube))
+// (process "some.wav" 30)
+// (every-frame
+// (with-primitive c
+// (rotate (vector (process-delta) 0 0))
+// (draw-cube)))
// EndFunctionDoc
-Scheme_Object *process(int argc, Scheme_Object **argv)
+Scheme_Object *process_delta(int argc, Scheme_Object **argv)
{
- MZ_GC_DECL_REG(1);
- MZ_GC_VAR_IN_REG(0, argv);
- MZ_GC_REG();
- if (!SCHEME_CHAR_STRINGP(argv[0])) scheme_wrong_type("process", "string", 0, argc, argv);
- char *wavname=scheme_utf8_encode_to_buffer(SCHEME_CHAR_STR_VAL(argv[0]),SCHEME_CHAR_STRLEN_VAL(argv[0]),NULL,0);
- if (Audio!=NULL)
- {
- Audio->Process(wavname);
+ Scheme_Object *ret = NULL;
+ MZ_GC_DECL_REG(1);
+ MZ_GC_VAR_IN_REG(0, ret);
+ MZ_GC_REG();
+ if (Audio != NULL) {
+ ret = scheme_make_double(Audio->ProcessDelta());
}
- MZ_GC_UNREG();
- return scheme_void;
+ MZ_GC_UNREG();
+ return ret;
+}
+
+// StartFunctionDoc-en
+// processing
+// Returns: bool
+// Description:
+// Returns #t if a wav file is being digested, so you can know when it has
+// reached the end.
+// Example:
+// (process "some.wav" 30)
+// (start-framedump "some" "ppm")
+// (every-frame (if (processing) (animate) (end-framedump)))
+// EndFunctionDoc
+
+Scheme_Object *processing(int argc, Scheme_Object **argv)
+{
+ Scheme_Object *ret = NULL;
+ MZ_GC_DECL_REG(1);
+ MZ_GC_VAR_IN_REG(0, ret);
+ MZ_GC_REG();
+ ret = scheme_false;
+ if (Audio != NULL) {
+ if (Audio->IsProcessing()) ret = scheme_true;
+ }
+ MZ_GC_UNREG();
+ return ret;
}
// StartFunctionDoc-en
@@ -564,7 +639,11 @@ Scheme_Object *scheme_reload(Scheme_Env *env)
scheme_add_global("gh", scheme_make_prim_w_arity(get_harmonic, "gh", 1, 1), menv);
scheme_add_global("ga", scheme_make_prim_w_arity(get_audio, "ga", 0, 0), menv);
scheme_add_global("gain", scheme_make_prim_w_arity(gain, "gain", 1, 1), menv);
- scheme_add_global("process", scheme_make_prim_w_arity(process, "process", 1, 1), menv);
+ scheme_add_global("process", scheme_make_prim_w_arity(process, "process", 2, 2), menv);
+ scheme_add_global("stop-process", scheme_make_prim_w_arity(stop_process, "stop-process", 0, 0), menv);
+ scheme_add_global("process-time", scheme_make_prim_w_arity(process_time, "process-time", 0, 0), menv);
+ scheme_add_global("process-delta", scheme_make_prim_w_arity(process_delta, "process-delta", 0, 0), menv);
+ scheme_add_global("processing", scheme_make_prim_w_arity(processing, "processing", 0, 0), menv);
scheme_add_global("smoothing-bias", scheme_make_prim_w_arity(smoothing_bias, "smoothing-bias", 1, 1), menv);
scheme_add_global("update-audio", scheme_make_prim_w_arity(update_audio, "update-audio", 0, 0), menv);
scheme_add_global("set-num-frequency-bins", scheme_make_prim_w_arity(set_num_frequency_bins, "set-num-frequency-bins", 1, 1), menv);
diff --git a/modules/fluxus-audio/src/JackClient.h b/modules/fluxus-audio/src/JackClient.h
index 7725024..93782d9 100644
--- a/modules/fluxus-audio/src/JackClient.h
+++ b/modules/fluxus-audio/src/JackClient.h
@@ -51,6 +51,7 @@ public:
void SetOutputBuf(int ID, float* s);
int AddInputPort();
int AddOutputPort();
+ int GetSamplerate() { if (m_Client != NULL) return jack_get_sample_rate(m_Client); return 0; }
protected:
JackClient();
