Added FFT sound visualization

CaseLights/AppDelegate.h

@@ -15,10 +15,17 @@
 
 @interface AppDelegate : NSObject <NSApplicationDelegate, EZMicrophoneDelegate>
 
+@property (weak) IBOutlet NSApplication *application;
+
+@property (strong) EZMicrophone *microphone;
+
 - (void)clearDisplayUI;
 - (void)updateDisplayUI:(NSArray *)displayIDs;
 
 - (void)setLightsR:(unsigned char)r G:(unsigned char)g B:(unsigned char)b;
 
++ (double)map:(double)val FromMin:(double)fmin FromMax:(double)fmax ToMin:(double)tmin ToMax:(double)tmax;
++ (void)convertH:(double)h S:(double)s V:(double)v toR:(unsigned char *)r G:(unsigned char *)g B:(unsigned char *)b;
+
 @end
 

CaseLights/AppDelegate.m

@@ -64,7 +64,6 @@
 @interface AppDelegate ()
 
 @property (weak) IBOutlet NSMenu *statusMenu;
-@property (weak) IBOutlet NSApplication *application;
 
 @property (weak) IBOutlet NSMenu *menuColors;
 @property (weak) IBOutlet NSMenu *menuAnimations;
@@ -89,7 +88,6 @@
 @property (strong) NSTimer *animation;
 @property (strong) Serial *serial;
 @property (strong) NSMenuItem *lastLEDMode;
-@property (strong) EZMicrophone *microphone;
 
 @end
 
@@ -880,14 +878,6 @@
     [application orderFrontStandardAboutPanel:self];
 }
 
-- (void)updateBuffer:(float *)buffer withBufferSize:(UInt32)bufferSize {
-    if (microphone == nil) {
-        return; // Old buffer from before we changed mode
-    }
-    
-    [AudioVisualizer updateBuffer:buffer withBufferSize:bufferSize];
-}
-
 // ------------------------------------------------------
 // ----------------- Microphone Delegate ----------------
 // ------------------------------------------------------
@@ -903,8 +893,12 @@
     // EZAudioPlot, EZAudioPlotGL, or whatever visualization you would like to do with
     // the microphone data.
     dispatch_async(dispatch_get_main_queue(),^{
+        if (weakSelf.microphone == nil) {
+            return;
+        }
+        
         // buffer[0] = left channel, buffer[1] = right channel
-        [weakSelf updateBuffer:buffer[0] withBufferSize:bufferSize];
+        [AudioVisualizer updateBuffer:buffer[0] withBufferSize:bufferSize];
     });
 }
 
@@ -956,14 +950,14 @@
     if ([GPUStats getGPUUsage:&usage freeVRAM:&freeVRAM usedVRAM:&usedVRAM] != 0) {
         NSLog(@"Error reading GPU information\n");
     } else {
-        double h = [self map:[usage doubleValue] FromMin:0.0 FromMax:100.0 ToMin:GPU_COLOR_MIN ToMax:GPU_COLOR_MAX];
+        double h = [AppDelegate map:[usage doubleValue] FromMin:0.0 FromMax:100.0 ToMin:GPU_COLOR_MIN ToMax:GPU_COLOR_MAX];
         
 #ifdef DEBUG
         NSLog(@"GPU Usage: %.3f%%\n", [usage doubleValue]);
 #endif
         
         unsigned char r, g, b;
-        [self convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
+        [AppDelegate convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
         [self setLightsR:r G:g B:b];
     }
 }
@@ -975,14 +969,14 @@
     if ([GPUStats getGPUUsage:&usage freeVRAM:&freeVRAM usedVRAM:&usedVRAM] != 0) {
         NSLog(@"Error reading GPU information\n");
     } else {
-        double h = [self map:[freeVRAM doubleValue] FromMin:0.0 FromMax:([freeVRAM doubleValue] + [usedVRAM doubleValue]) ToMin:RAM_COLOR_MIN ToMax:RAM_COLOR_MAX];
+        double h = [AppDelegate map:[freeVRAM doubleValue] FromMin:0.0 FromMax:([freeVRAM doubleValue] + [usedVRAM doubleValue]) ToMin:RAM_COLOR_MIN ToMax:RAM_COLOR_MAX];
         
 #ifdef DEBUG
         NSLog(@"VRAM %.2fGB Free + %.2fGB Used = %.2fGB mapped to color %.2f!\n", [freeVRAM doubleValue] / (1024.0 * 1024.0 * 1024.0), [usedVRAM doubleValue] / (1024.0 * 1024.0 * 1024.0), ([freeVRAM doubleValue] + [usedVRAM doubleValue]) / (1024.0 * 1024.0 * 1024.0), h);
 #endif
         
         unsigned char r, g, b;
-        [self convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
+        [AppDelegate convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
         [self setLightsR:r G:g B:b];
     }
 }
@@ -990,28 +984,28 @@
 - (void)visualizeCPUUsage:(NSTimer *)timer {
     JSKMCPUUsageInfo cpuUsageInfo = [JSKSystemMonitor systemMonitor].cpuUsageInfo;
     
-    double h = [self map:cpuUsageInfo.usage FromMin:0.0 FromMax:100.0 ToMin:CPU_COLOR_MIN ToMax:CPU_COLOR_MAX];
+    double h = [AppDelegate map:cpuUsageInfo.usage FromMin:0.0 FromMax:100.0 ToMin:CPU_COLOR_MIN ToMax:CPU_COLOR_MAX];
     
 #ifdef DEBUG
     NSLog(@"CPU Usage: %.3f%%\n", cpuUsageInfo.usage);
 #endif
     
     unsigned char r, g, b;
-    [self convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
+    [AppDelegate convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
     [self setLightsR:r G:g B:b];
 }
 
 - (void)visualizeRAMUsage:(NSTimer *)timer {
     JSKMMemoryUsageInfo memoryUsageInfo = [JSKSystemMonitor systemMonitor].memoryUsageInfo;
     
-    double h = [self map:memoryUsageInfo.freeMemory FromMin:0.0 FromMax:(memoryUsageInfo.usedMemory + memoryUsageInfo.freeMemory) ToMin:RAM_COLOR_MIN ToMax:RAM_COLOR_MAX];
+    double h = [AppDelegate map:memoryUsageInfo.freeMemory FromMin:0.0 FromMax:(memoryUsageInfo.usedMemory + memoryUsageInfo.freeMemory) ToMin:RAM_COLOR_MIN ToMax:RAM_COLOR_MAX];
     
 #ifdef DEBUG
     NSLog(@"RAM %.2fGB Free + %.2fGB Used = %.2fGB mapped to color %.2f!\n", memoryUsageInfo.freeMemory / (1024.0 * 1024.0 * 1024.0), memoryUsageInfo.usedMemory / (1024.0 * 1024.0 * 1024.0), (memoryUsageInfo.freeMemory + memoryUsageInfo.usedMemory) / (1024.0 * 1024.0 * 1024.0), h);
 #endif
     
     unsigned char r, g, b;
-    [self convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
+    [AppDelegate convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
     [self setLightsR:r G:g B:b];
 }
 
@@ -1031,14 +1025,14 @@
         temp = GPU_TEMP_MIN;
     }
     
-    double h = [self map:temp FromMin:GPU_TEMP_MIN FromMax:GPU_TEMP_MAX ToMin:GPU_COLOR_MIN ToMax:GPU_COLOR_MAX];
+    double h = [AppDelegate map:temp FromMin:GPU_TEMP_MIN FromMax:GPU_TEMP_MAX ToMin:GPU_COLOR_MIN ToMax:GPU_COLOR_MAX];
     
 #ifdef DEBUG
     NSLog(@"GPU Temp %.2f mapped to color %.2f!\n", temp, h);
 #endif
     
     unsigned char r, g, b;
-    [self convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
+    [AppDelegate convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
     [self setLightsR:r G:g B:b];
 }
 
@@ -1058,14 +1052,14 @@
         temp = CPU_TEMP_MIN;
     }
     
-    double h = [self map:temp FromMin:CPU_TEMP_MIN FromMax:CPU_TEMP_MAX ToMin:CPU_COLOR_MIN ToMax:CPU_COLOR_MAX];
+    double h = [AppDelegate map:temp FromMin:CPU_TEMP_MIN FromMax:CPU_TEMP_MAX ToMin:CPU_COLOR_MIN ToMax:CPU_COLOR_MAX];
     
 #ifdef DEBUG
     NSLog(@"CPU Temp %.2f mapped to color %.2f!\n", temp, h);
 #endif
     
     unsigned char r, g, b;
-    [self convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
+    [AppDelegate convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
     [self setLightsR:r G:g B:b];
 }
 
@@ -1103,7 +1097,7 @@
     }
     
     unsigned char r, g, b;
-    [self convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
+    [AppDelegate convertH:h S:1.0 V:1.0 toR:&r G:&g B:&b];
     [self setLightsR:r G:g B:b];
 }
 
@@ -1115,12 +1109,12 @@
 // --------------------- Utilities ---------------------
 // -----------------------------------------------------
 
-- (double)map:(double)val FromMin:(double)fmin FromMax:(double)fmax ToMin:(double)tmin ToMax:(double)tmax {
++ (double)map:(double)val FromMin:(double)fmin FromMax:(double)fmax ToMin:(double)tmin ToMax:(double)tmax {
     double norm = (val - fmin) / (fmax - fmin);
     return (norm * (tmax - tmin)) + tmin;
 }
 
-- (void)convertH:(double)h S:(double)s V:(double)v toR:(unsigned char *)r G:(unsigned char *)g B:(unsigned char *)b {
++ (void)convertH:(double)h S:(double)s V:(double)v toR:(unsigned char *)r G:(unsigned char *)g B:(unsigned char *)b {
     // Adapted from:
     // https://gist.github.com/hdznrrd/656996
     

CaseLights/AudioVisualizer.m

@@ -2,14 +2,48 @@
 //  AudioVisualizer.m
 //  CaseLights
 //
+//  Based on the ideas in:
+//  http://archive.gamedev.net/archive/reference/programming/features/beatdetection/
+//
+//  The detected sound frequency of beats will be mapped to the hue of the resulting color,
+//  the variance of the beat is mapped to the brightness of the color. The colors
+//  of all detected beats will be added together to form the final displayed color.
+//
 //  Created by Thomas Buck on 01.01.16.
 //  Copyright © 2016 xythobuz. All rights reserved.
 //
 
+// Enabling this will cause crashes when changing audio input
+// devices while the app is running. Select it before enabling.
+#define DEBUG_PLOT_FFT
+//#define DEBUG_PLOT_FFT_RAW
+
+#define DEBUG_LOG_BEATS
+
 #import "AudioVisualizer.h"
 #import "AppDelegate.h"
 
+#import "EZAudioFFT.h"
+
+#ifdef DEBUG_PLOT_FFT
+#import "EZAudioPlot.h"
+#endif
+
+// Parameters for fine-tuning beat detection
+#define FFT_BUCKET_COUNT 64
+#define FFT_BUCKET_HISTORY 43
+#define FFT_C_FACTOR 4.2
+#define FFT_V0_FACTOR 0.000015
+#define FFT_MAX_V0_COLOR 0.0002
+#define FFT_COLOR_DECAY 0.99
+
+// Factors for nicer debug display
+#define FFT_DEBUG_RAW_FACTOR 42.0
+#define FFT_DEBUG_FACTOR 230.0
+
 static AppDelegate *appDelegate = nil;
+static EZAudioFFT *fft = nil;
+static int maxBufferSize = 0;
 
 @implementation AudioVisualizer
 
@@ -18,7 +52,159 @@ static AppDelegate *appDelegate = nil;
 }
 
 + (void)updateBuffer:(float *)buffer withBufferSize:(UInt32)bufferSize {
+    // Create Fast Fourier Transformation object
+    if (fft == nil) {
+        maxBufferSize = bufferSize;
+        fft = [EZAudioFFT fftWithMaximumBufferSize:maxBufferSize sampleRate:appDelegate.microphone.audioStreamBasicDescription.mSampleRate];
+        
+#ifdef DEBUG
+        NSLog(@"Created FFT with max. freq.: %.2f\n", appDelegate.microphone.audioStreamBasicDescription.mSampleRate / 2);
+#endif
+    }
+    
+    // Check for changing buffer sizes
+    if (bufferSize > maxBufferSize) {
+        NSLog(@"Buffer Size changed?! %d != %d\n", maxBufferSize, bufferSize);
+        maxBufferSize = bufferSize;
+        fft = [EZAudioFFT fftWithMaximumBufferSize:maxBufferSize sampleRate:appDelegate.microphone.audioStreamBasicDescription.mSampleRate];
+    }
+    
+    [fft computeFFTWithBuffer:buffer withBufferSize:bufferSize];
+    
+    static float history[FFT_BUCKET_COUNT][FFT_BUCKET_HISTORY];
+    static int nextHistory =  0;
+    static int samplesPerBucket = 0;
+    
+    // Initialize static variables
+    if (samplesPerBucket == 0) {
+        samplesPerBucket = bufferSize / FFT_BUCKET_COUNT;
+        for (int i = 0; i < FFT_BUCKET_COUNT; i++) {
+            for (int j = 0; j < FFT_BUCKET_HISTORY; j++) {
+                history[i][j] = 0.5f;
+            }
+        }
+    }
+    
+    // Split FFT output into a small number of 'buckets' or 'bins' and add to circular history buffer
+    for (int i = 0; i < FFT_BUCKET_COUNT; i++) {
+        float sum = 0.0f;
+        for (int j = 0; j < samplesPerBucket; j++) {
+            sum += fft.fftData[(i + samplesPerBucket) + j];
+        }
+        history[i][nextHistory] = sum / samplesPerBucket;
+    }
+    
+#ifdef DEBUG_PLOT_FFT
+    int beatCount = 0;
+#endif
+    
+    static unsigned char lastRed = 0, lastGreen = 0, lastBlue = 0;
+    lastRed = lastRed * FFT_COLOR_DECAY;
+    lastGreen = lastGreen * FFT_COLOR_DECAY;
+    lastBlue = lastBlue * FFT_COLOR_DECAY;
+    
+    // Check for any beats
+    for (int i = 0; i < FFT_BUCKET_COUNT; i++) {
+        float average = 0.0f;
+        for (int j = 0; j < FFT_BUCKET_HISTORY; j++) {
+            average += history[i][j];
+        }
+        average /= FFT_BUCKET_HISTORY;
+        float v = 0.0f;
+        for (int j = 0; j < FFT_BUCKET_HISTORY; j++) {
+            float tmp = history[i][j] - average;
+            tmp *= tmp;
+            v += tmp;
+        }
+        v /= FFT_BUCKET_HISTORY;
+        if ((history[i][nextHistory] > (FFT_C_FACTOR * average)) && (v > FFT_V0_FACTOR)) {
+            // Found a beat on this frequency band, map to a single color
+            if (v < FFT_V0_FACTOR) v = FFT_V0_FACTOR;
+            if (v > FFT_MAX_V0_COLOR) v = FFT_MAX_V0_COLOR;
+            float bright = [AppDelegate map:v FromMin:FFT_V0_FACTOR FromMax:FFT_MAX_V0_COLOR ToMin:0.0 ToMax:100.0];
+            float hue = [AppDelegate map:i FromMin:0.0 FromMax:FFT_BUCKET_COUNT ToMin:0.0 ToMax:360.0];
+            unsigned char r, g, b;
+            [AppDelegate convertH:hue S:1.0 V:bright toR:&r G:&g B:&b];
+            
+            // Blend with last color using averaging
+            int tmpR = (lastRed + r) / 2;
+            int tmpG = (lastGreen + g) / 2;
+            int tmpB = (lastBlue + b) / 2;
+            lastRed = tmpR;
+            lastGreen = tmpG;
+            lastBlue = tmpB;
+            
+#ifdef DEBUG_LOG_BEATS
+            NSLog(@"Beat in %d with c: %f v: %f", i, (history[i][nextHistory] / average), v);
+#endif
+#ifdef DEBUG_PLOT_FFT
+            beatCount++;
+#endif
+        }
+    }
+    
+    [appDelegate setLightsR:lastRed G:lastGreen B:lastBlue];
+
+#ifdef DEBUG_PLOT_FFT
+    static NSWindow *window = nil;
+    static EZAudioPlot *plot = nil;
+    static NSTextField *label = nil;
+    if ((window == nil) || (plot == nil) || (label == nil)) {
+        NSRect frame = NSMakeRect(450, 300, 600, 400);
+        window = [[NSWindow alloc] initWithContentRect:frame
+                                             styleMask:NSClosableWindowMask | NSTitledWindowMask | NSBorderlessWindowMask
+                                               backing:NSBackingStoreBuffered
+                                                 defer:NO];
+        [window setTitle:@"Debug FFT"];
+        
+        plot = [[EZAudioPlot alloc] initWithFrame:window.contentView.frame];
+        plot.color = [NSColor whiteColor];
+        plot.shouldOptimizeForRealtimePlot = NO; // Not working with 'YES' here?!
+        plot.shouldFill = YES;
+        plot.shouldCenterYAxis = NO;
+        plot.shouldMirror = NO;
+        plot.plotType = EZPlotTypeBuffer;
+        [window.contentView addSubview:plot];
+        
+        label = [[NSTextField alloc] initWithFrame:NSMakeRect(0, 380, 600, 20)];
+        [label setTextColor:[NSColor whiteColor]];
+        [label setEditable:NO];
+        [label setBezeled:NO];
+        [label setDrawsBackground:NO];
+        [label setSelectable:NO];
+        [label setStringValue:@"-"];
+        [window.contentView addSubview:label];
+        
+        [window makeKeyAndOrderFront:appDelegate.application];
+        NSLog(@"Created debugging FFT Plot window...\n");
+    }
+    
+    // Scale so we can see something
+# ifdef DEBUG_PLOT_FFT_RAW
+    memcpy(buffer, fft.fftData, bufferSize * sizeof(float));
+    for (UInt32 i = 0; i < bufferSize; i++) {
+        buffer[i] *= FFT_DEBUG_RAW_FACTOR;
+# else
+    for (int i = 0; i < FFT_BUCKET_COUNT; i++) {
+        buffer[i] = history[i][nextHistory];
+    }
+    for (UInt32 i = 0; i < FFT_BUCKET_COUNT; i++) {
+        buffer[i] *= FFT_DEBUG_FACTOR;
+# endif
+        if (buffer[i] > 1.0f) buffer[i] = 1.0f;
+        if (buffer[i] < -1.0f) buffer[i] = -1.0f;
+    }
+    [plot updateBuffer:buffer withBufferSize:bufferSize];
     
+    [window setBackgroundColor:[NSColor colorWithCalibratedRed:lastRed / 255.0 green:lastGreen / 255.0 blue:lastBlue / 255.0 alpha:1.0]];
+    [label setStringValue:[NSString stringWithFormat:@"Beats: %d", beatCount]];
+#endif
+        
+    // Point to next history buffer
+    nextHistory++;
+    if (nextHistory >= FFT_BUCKET_HISTORY) {
+        nextHistory = 0;
+    }
 }
 
 @end

CaseLights/Info.plist

@@ -21,7 +21,7 @@
 	<key>CFBundleSignature</key>
 	<string>????</string>
 	<key>CFBundleVersion</key>
-	<string>182</string>
+	<string>359</string>
 	<key>LSApplicationCategoryType</key>
 	<string>public.app-category.utilities</string>
 	<key>LSMinimumSystemVersion</key>

CaseLights/Serial.m

@@ -9,6 +9,8 @@
 //  Copyright © 2015 xythobuz. All rights reserved.
 //
 
+//#define DEBUG_TEXT
+
 #import <Cocoa/Cocoa.h>
 #import <IOKit/IOKitLib.h>
 #import <IOKit/serial/IOSerialKeys.h>
@@ -155,7 +157,7 @@
     const char *data = [string UTF8String];
     size_t length = strlen(data);
     
-#ifdef DEBUG
+#ifdef DEBUG_TEXT
     NSLog(@"Sending string %s", data);
 #endif
     

README.md

@@ -22,6 +22,8 @@ CaseLights is only visible in the system menu bar. You can enable or disable the
 
 You can also select one of the displays connected to the Host machine. The CaseLights App will then create a Screenshot of this display 10-times per second and calculate the average color to display it on the RGB LEDs.
 
+CaseLights is also able to visualize sound coming from a system audio input. To be able to directly visualize the system sound output, install [Soundflower](https://github.com/mattingalls/Soundflower) and create a Multi-Output-Device in the Mac `Audio Midi Setup.app` consisting of `Soundflower (2ch)` and your normally used output device. Set this device as audio output. Then, in CaseLights, select `Soundflower (2ch)` as input.
+
 ## Working with Git Submodules
 
 To clone this repository, enter the following: