Преглед на файлове

Attempted to implement Screen Capture with AVFoundation framework.

Thomas Buck преди 7 години
родител
ревизия
f4e8c7246d
No account linked to committer's email address
променени са 3 файла, в които са добавени 232 реда и са изтрити 71 реда
  1. 3
    1
      DisplayBacklight/AppDelegate.h
  2. 228
    69
      DisplayBacklight/AppDelegate.m
  3. 1
    1
      DisplayBacklight/Info.plist

+ 3
- 1
DisplayBacklight/AppDelegate.h Целия файл

@@ -7,10 +7,11 @@
7 7
 //
8 8
 
9 9
 #import <Cocoa/Cocoa.h>
10
+#import <AVFoundation/AVFoundation.h>
10 11
 
11 12
 @class Serial;
12 13
 
13
-@interface AppDelegate : NSObject <NSApplicationDelegate>
14
+@interface AppDelegate : NSObject <NSApplicationDelegate, AVCaptureVideoDataOutputSampleBufferDelegate>
14 15
 
15 16
 @property (weak) IBOutlet NSApplication *application;
16 17
 
@@ -23,6 +24,7 @@
23 24
 // between multiple displays with the same resolution
24 25
 struct DisplayAssignment {
25 26
     int width, height;
27
+    int shown;
26 28
 };
27 29
 
28 30
 struct LEDStrand {

+ 228
- 69
DisplayBacklight/AppDelegate.m Целия файл

@@ -25,9 +25,11 @@
25 25
 
26 26
 // Identify your displays here. Currently they're only distinguished by their resolution.
27 27
 // The ID will be the index in the list, so the first entry is display 0 and so on.
28
+// The third parameter is used internally for keeping track of the visualized displays,
29
+// simply set it to 0.
28 30
 struct DisplayAssignment displays[] = {
29
-    { 1920, 1080 },
30
-    {  900, 1600 }
31
+    { 1920, 1080, 0 },
32
+    {  900, 1600, 0 }
31 33
 };
32 34
 
33 35
 // This defines the orientation and placement of your strands and is the most important part.
@@ -43,7 +45,7 @@ struct DisplayAssignment displays[] = {
43 45
 // if your strand does not span the whole length of this screen edge.
44 46
 //
45 47
 // For example, this is my personal dual-monitor home setup. The strand starts at the 0
46
-// in the bottom-right of D1, then goes left arount D1, and from there around D2 back
48
+// in the bottom-right of D1, then goes left around D1, and from there around D2 back
47 49
 // to the start.
48 50
 //
49 51
 //                                       29
@@ -82,7 +84,7 @@ struct LEDStrand strands[] = {
82 84
 // but will probably cause high CPU-Usage.
83 85
 // (Run-Time of the algorithm is ignored here, so real speed will be
84 86
 // slightly lower.)
85
-#define DISPLAY_DELAY (1.0 / 30.0)
87
+#define DISPLAY_DELAY (1.0 / 10.0)
86 88
 
87 89
 // How many pixels to skip when calculating the average color.
88 90
 // Slightly increases performance and doesn't really alter the result.
@@ -98,7 +100,7 @@ struct LEDStrand strands[] = {
98 100
 #define PREF_TURNED_ON @"IsEnabled"
99 101
 
100 102
 // If this is defined it will print the FPS every DEBUG_PRINT_FPS seconds
101
-//#define DEBUG_PRINT_FPS 10
103
+#define DEBUG_PRINT_FPS 2.5
102 104
 
103 105
 // ------------------------ Config ends here ------------------------
104 106
 
@@ -117,6 +119,8 @@ struct LEDStrand strands[] = {
117 119
 @property (strong) Serial *serial;
118 120
 @property (strong) NSArray *lastDisplayIDs;
119 121
 @property (assign) BOOL restartAmbilight;
122
+@property (strong) NSArray *captureSessions;
123
+@property (strong) NSLock *lock;
120 124
 
121 125
 @end
122 126
 
@@ -127,11 +131,14 @@ struct LEDStrand strands[] = {
127 131
 @synthesize brightnessItem, brightnessSlider, brightnessLabel;
128 132
 @synthesize statusItem, statusImage, lastDisplayIDs;
129 133
 @synthesize timer, serial, restartAmbilight;
134
+@synthesize captureSessions, lock;
130 135
 
131 136
 - (void)applicationDidFinishLaunching:(NSNotification *)aNotification {
132 137
     serial = [[Serial alloc] init];
138
+    lock = [[NSLock alloc] init];
133 139
     timer = nil;
134 140
     restartAmbilight = NO;
141
+    captureSessions = nil;
135 142
     
136 143
     // Set default configuration values
137 144
     NSUserDefaults *store = [NSUserDefaults standardUserDefaults];
@@ -201,22 +208,14 @@ struct LEDStrand strands[] = {
201 208
         }
202 209
     }
203 210
     
211
+    // Enumerate displays and start ambilight if required
204 212
     [Screenshot init:self];
205
-    lastDisplayIDs = [Screenshot listDisplays];
206
-    
207
-    if (ambilightIsOn) {
208
-        timer = [NSTimer scheduledTimerWithTimeInterval:DISPLAY_DELAY target:self selector:@selector(visualizeDisplay:) userInfo:nil repeats:NO];
209
-        
210
-        [buttonAmbilight setState:NSOnState];
211
-    }
213
+    restartAmbilight = ambilightIsOn;
214
+    [self newDisplayList:[Screenshot listDisplays]];
212 215
 }
213 216
 
214 217
 - (void)applicationWillTerminate:(NSNotification *)aNotification {
215
-    // Stop previous timer setting
216
-    if (timer != nil) {
217
-        [timer invalidate];
218
-        timer = nil;
219
-    }
218
+    [self stopAmbilightTimer];
220 219
     
221 220
     // Remove display callback
222 221
     [Screenshot close:self];
@@ -228,6 +227,39 @@ struct LEDStrand strands[] = {
228 227
     }
229 228
 }
230 229
 
230
+BOOL timerRunning = NO;
231
+
232
+- (void)startAmbilightTimer {
233
+    //timer = [NSTimer scheduledTimerWithTimeInterval:DISPLAY_DELAY target:self selector:@selector(visualizeDisplay:) userInfo:nil repeats:NO];
234
+    
235
+    timerRunning = YES;
236
+    if (captureSessions != nil) {
237
+        for (int i = 0; i < [captureSessions count]; i++) {
238
+            [[captureSessions objectAtIndex:i] startRunning];
239
+        }
240
+    }
241
+}
242
+
243
+- (void)stopAmbilightTimer {
244
+    // Stop previous timer setting
245
+    //if (timer != nil) {
246
+    //    [timer invalidate];
247
+    //    timer = nil;
248
+    //}
249
+    
250
+    timerRunning = NO;
251
+    if (captureSessions != nil) {
252
+        for (int i = 0; i < [captureSessions count]; i++) {
253
+            [[captureSessions objectAtIndex:i] stopRunning];
254
+        }
255
+    }
256
+}
257
+
258
+- (BOOL)isAmbilightRunning {
259
+    //return (timer != nil) ? YES : NO;
260
+    return timerRunning;
261
+}
262
+
231 263
 - (IBAction)relistSerialPorts:(id)sender {
232 264
     // Refill port list
233 265
     NSArray *ports = [Serial listSerialPorts];
@@ -265,11 +297,7 @@ struct LEDStrand strands[] = {
265 297
         [serial closePort];
266 298
     }
267 299
     
268
-    // Stop previous timer setting
269
-    if (timer != nil) {
270
-        [timer invalidate];
271
-        timer = nil;
272
-    }
300
+    [self stopAmbilightTimer];
273 301
     
274 302
     // Turn off ambilight button
275 303
     [buttonAmbilight setState:NSOffState];
@@ -284,13 +312,7 @@ struct LEDStrand strands[] = {
284 312
 - (IBAction)toggleAmbilight:(NSMenuItem *)sender {
285 313
     if ([sender state] == NSOnState) {
286 314
         [sender setState:NSOffState];
287
-        
288
-        // Stop previous timer setting
289
-        if (timer != nil) {
290
-            [timer invalidate];
291
-            timer = nil;
292
-        }
293
-        
315
+        [self stopAmbilightTimer];
294 316
         [self sendNullFrame];
295 317
         
296 318
         // Store state
@@ -299,8 +321,7 @@ struct LEDStrand strands[] = {
299 321
         [store synchronize];
300 322
     } else {
301 323
         [sender setState:NSOnState];
302
-        
303
-        timer = [NSTimer scheduledTimerWithTimeInterval:DISPLAY_DELAY target:self selector:@selector(visualizeDisplay:) userInfo:nil repeats:NO];
324
+        [self startAmbilightTimer];
304 325
         
305 326
         // Store state
306 327
         NSUserDefaults *store = [NSUserDefaults standardUserDefaults];
@@ -310,24 +331,64 @@ struct LEDStrand strands[] = {
310 331
 }
311 332
 
312 333
 - (void)stopAmbilight {
313
-    restartAmbilight = NO;
314
-    if (timer != nil) {
315
-        restartAmbilight = YES;
316
-        [timer invalidate];
317
-        timer = nil;
318
-        
319
-        [buttonAmbilight setState:NSOffState];
320
-    }
334
+    restartAmbilight = [self isAmbilightRunning];
335
+    [buttonAmbilight setState:NSOffState];
336
+    [self stopAmbilightTimer];
321 337
 }
322 338
 
323 339
 - (void)newDisplayList:(NSArray *)displayIDs {
324 340
     lastDisplayIDs = displayIDs;
325 341
     
342
+    // Create capturing sessions for each display
343
+    AVCaptureSession *sessions[[displayIDs count]];
344
+    for (int i = 0; i < [displayIDs count]; i++) {
345
+        sessions[i] = [[AVCaptureSession alloc] init];
346
+        [sessions[i] beginConfiguration];
347
+        
348
+        if ([sessions[i] canSetSessionPreset:AVCaptureSessionPresetHigh]) {
349
+            // TODO could use other presets?
350
+            sessions[i].sessionPreset = AVCaptureSessionPresetHigh;
351
+        } else {
352
+            NSLog(@"Can't set preset for display %ld!", (long)[[displayIDs objectAtIndex:i] integerValue]);
353
+        }
354
+        
355
+        // Add Screen Capture input for this screen
356
+        AVCaptureScreenInput *input = [[AVCaptureScreenInput alloc] initWithDisplayID:[[displayIDs objectAtIndex:i] unsignedIntValue]];
357
+        [input setCapturesCursor:YES]; // Enable mouse cursor capturing (ToDo disable for performance?)
358
+        [input setMinFrameDuration:CMTimeMakeWithSeconds(DISPLAY_DELAY, 1000)]; // Set out target frame rate
359
+        
360
+        if ([sessions[i] canAddInput:input]) {
361
+            [sessions[i] addInput:input];
362
+        } else {
363
+            NSLog(@"Can't add screen grab input for display %ld!", (long)[[displayIDs objectAtIndex:i] integerValue]);
364
+        }
365
+        
366
+        // Add Screen Capture output into this object
367
+        AVCaptureVideoDataOutput *output = [[AVCaptureVideoDataOutput alloc] init];
368
+        [output setSampleBufferDelegate:self queue:dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)];
369
+        [output setVideoSettings:@{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA) }];
370
+        
371
+        NSArray *formats = [output availableVideoCVPixelFormatTypes];
372
+        for (int i = 0; i < [formats count]; i++) {
373
+            NSLog(@"Supported format: 0x%lX", (long)[[formats objectAtIndex:i] integerValue]);
374
+        }
375
+        
376
+        if ([sessions[i] canAddOutput:output]) {
377
+            [sessions[i] addOutput:output];
378
+        } else {
379
+            NSLog(@"Can't add screen grab output for display %ld!", (long)[[displayIDs objectAtIndex:i] integerValue]);
380
+        }
381
+        
382
+        [sessions[i] commitConfiguration];
383
+        
384
+        NSLog(@"Added output for display %ld", (long)[[displayIDs objectAtIndex:i] integerValue]);
385
+    }
386
+    captureSessions = [NSArray arrayWithObjects:sessions count:[displayIDs count]];
387
+    
326 388
     if (restartAmbilight) {
327 389
         restartAmbilight = NO;
328 390
         [buttonAmbilight setState:NSOnState];
329
-        
330
-        timer = [NSTimer scheduledTimerWithTimeInterval:DISPLAY_DELAY target:self selector:@selector(visualizeDisplay:) userInfo:nil repeats:NO];
391
+        [self startAmbilightTimer];
331 392
     }
332 393
 }
333 394
 
@@ -346,6 +407,8 @@ struct LEDStrand strands[] = {
346 407
 }
347 408
 
348 409
 - (void)sendLEDFrame {
410
+    //NSLog(@"New LED frame");
411
+    
349 412
     if ([serial isOpen]) {
350 413
         [serial sendString:MAGIC_WORD];
351 414
         [serial sendData:(char *)ledColorData withLength:(sizeof(ledColorData) / sizeof(ledColorData[0]))];
@@ -366,9 +429,10 @@ struct LEDStrand strands[] = {
366 429
 UInt8 ledColorData[LED_COUNT * 3];
367 430
 
368 431
 - (UInt32)calculateAverage:(unsigned char *)data Width:(NSInteger)width Height:(NSInteger)height SPP:(NSInteger)spp Alpha:(BOOL)alpha StartX:(NSInteger)startX StartY:(NSInteger)startY EndX:(NSInteger)endX EndY:(NSInteger)endY {
369
-    int redC = 0, greenC = 1, blueC = 2;
432
+    //int redC = 0, greenC = 1, blueC = 2;
433
+    int redC = 2, greenC = 1, blueC = 0;
370 434
     if (alpha) {
371
-        redC = 1; greenC = 2; blueC = 3;
435
+        //redC = 3; greenC = 2; blueC = 1;
372 436
     }
373 437
     
374 438
     NSInteger xa, xb, ya, yb;
@@ -409,6 +473,8 @@ UInt8 ledColorData[LED_COUNT * 3];
409 473
 }
410 474
 
411 475
 - (void)visualizeSingleDisplay:(NSInteger)disp Data:(unsigned char *)data Width:(unsigned long)width Height:(unsigned long)height SPP:(NSInteger)spp Alpha:(BOOL)alpha {
476
+    displays[disp].shown = 1;
477
+    
412 478
     for (int i = 0; i < (sizeof(strands) / sizeof(strands[0])); i++) {
413 479
         if (strands[i].display == disp) {
414 480
             // Walk the strand, calculating value for each LED
@@ -468,8 +534,123 @@ UInt8 ledColorData[LED_COUNT * 3];
468 534
             }
469 535
         }
470 536
     }
537
+    
538
+    int doneCount = 0;;
539
+    for (int i = 0; i < (sizeof(displays) / sizeof(displays[0])); i++) {
540
+        if (displays[i].shown != 0) {
541
+            doneCount++;
542
+        }
543
+    }
544
+    
545
+    if (doneCount >= (sizeof(displays) / sizeof(displays[0]))) {
546
+        [self sendLEDFrame];
547
+        
548
+        for (int i = 0; i < (sizeof(displays) / sizeof(displays[0])); i++) {
549
+            displays[i].shown = 0;
550
+        }
551
+    }
471 552
 }
472 553
 
554
+- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {
555
+    [lock lock];
556
+
557
+#ifdef DEBUG_PRINT_FPS
558
+    static NSInteger frameCount = 0;
559
+    static NSDate *lastPrintTime = nil;
560
+    if (lastPrintTime == nil) {
561
+        lastPrintTime = [NSDate date];
562
+    }
563
+#endif
564
+    
565
+    if (![self isAmbilightRunning]) {
566
+        [lock unlock];
567
+        return;
568
+    }
569
+
570
+    CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer);
571
+    CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
572
+    
573
+    //NSLog(@"W=%d H=%d", dimensions.width, dimensions.height);
574
+    
575
+    // Try to find the matching display id for the strand associations
576
+    for (int n = 0; n < (sizeof(displays) / sizeof(displays[0])); n++) {
577
+        if ((dimensions.width == displays[n].width) && (dimensions.height == displays[n].height)) {
578
+            //NSLog(@"Capture conversion for %d...", n);
579
+            
580
+            // Convert our frame to an NSImage
581
+            CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
582
+            CVPixelBufferLockBaseAddress(imageBuffer, 0);
583
+            
584
+            void *baseAddress = CVPixelBufferGetBaseAddress(imageBuffer);
585
+            size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
586
+            size_t width = CVPixelBufferGetWidth(imageBuffer);
587
+            size_t height = CVPixelBufferGetHeight(imageBuffer);
588
+            
589
+            CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
590
+            CGContextRef context = CGBitmapContextCreate(baseAddress, width, height, 8, bytesPerRow,
591
+                                                         colorSpace, kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Little);
592
+            if (context == nil) {
593
+                NSLog(@"Error creating context!");
594
+                break;
595
+            }
596
+            
597
+            CGImageRef quartzImage = CGBitmapContextCreateImage(context);
598
+            CVPixelBufferUnlockBaseAddress(imageBuffer,0);
599
+            
600
+            CGContextRelease(context);
601
+            CGColorSpaceRelease(colorSpace);
602
+            
603
+            NSBitmapImageRep *image = [[NSBitmapImageRep alloc] initWithCGImage:quartzImage];
604
+            CGImageRelease(quartzImage);
605
+            
606
+            [self visualizeThisImage:image];
607
+            break;
608
+        }
609
+    }
610
+    
611
+#ifdef DEBUG_PRINT_FPS
612
+    frameCount++;
613
+    NSDate *now = [NSDate date];
614
+    NSTimeInterval interval = [now timeIntervalSinceDate:lastPrintTime];
615
+    if (interval >= DEBUG_PRINT_FPS) {
616
+        NSLog(@"FPS: ~%.2f / %lu = %.2f", frameCount / interval,
617
+              (sizeof(displays) / sizeof(displays[0])), frameCount / interval / (sizeof(displays) / sizeof(displays[0])));
618
+        frameCount = 0;
619
+        lastPrintTime = now;
620
+    }
621
+#endif
622
+    
623
+    [lock unlock];
624
+}
625
+
626
+- (void)visualizeThisImage:(NSBitmapImageRep *)screen {
627
+    unsigned long width = [screen pixelsWide];
628
+    unsigned long height = [screen pixelsHigh];
629
+    
630
+    // Ensure we can handle the format of this display
631
+    NSInteger spp = [screen samplesPerPixel];
632
+    if (((spp != 3) && (spp != 4)) || ([screen isPlanar] == YES) || ([screen numberOfPlanes] != 1)) {
633
+        NSLog(@"Unknown image format for (%ld, %c, %ld)!\n", (long)spp, ([screen isPlanar] == YES) ? 'p' : 'n', (long)[screen numberOfPlanes]);
634
+        return;
635
+    }
636
+    
637
+    // Find out how the color components are ordered
638
+    BOOL alpha = NO;
639
+    if ([screen bitmapFormat] & NSAlphaFirstBitmapFormat) {
640
+        alpha = YES;
641
+    }
642
+    
643
+    // Try to find the matching display id for the strand associations
644
+    for (int n = 0; n < (sizeof(displays) / sizeof(displays[0])); n++) {
645
+        if ((width == displays[n].width) && (height == displays[n].height)) {
646
+            unsigned char *data = [screen bitmapData];
647
+            [self visualizeSingleDisplay:n Data:data Width:width Height:height SPP:spp Alpha:alpha];
648
+            return;
649
+        }
650
+    }
651
+}
652
+
653
+/*
473 654
 - (void)visualizeDisplay:(NSTimer *)time {
474 655
 #ifdef DEBUG_PRINT_FPS
475 656
     static NSInteger frameCount = 0;
@@ -484,30 +665,7 @@ UInt8 ledColorData[LED_COUNT * 3];
484 665
     // Create a Screenshot for all connected displays
485 666
     for (NSInteger i = 0; i < [lastDisplayIDs count]; i++) {
486 667
         NSBitmapImageRep *screen = [Screenshot screenshot:[lastDisplayIDs objectAtIndex:i]];
487
-        unsigned long width = [screen pixelsWide];
488
-        unsigned long height = [screen pixelsHigh];
489
-        
490
-        // Ensure we can handle the format of this display
491
-        NSInteger spp = [screen samplesPerPixel];
492
-        if (((spp != 3) && (spp != 4)) || ([screen isPlanar] == YES) || ([screen numberOfPlanes] != 1)) {
493
-            NSLog(@"Unknown image format for %ld (%ld, %c, %ld)!\n", (long)i, (long)spp, ([screen isPlanar] == YES) ? 'p' : 'n', (long)[screen numberOfPlanes]);
494
-            continue;
495
-        }
496
-        
497
-        // Find out how the color components are ordered
498
-        BOOL alpha = NO;
499
-        if ([screen bitmapFormat] & NSAlphaFirstBitmapFormat) {
500
-            alpha = YES;
501
-        }
502
-        
503
-        // Try to find the matching display id for the strand associations
504
-        for (int n = 0; n < (sizeof(displays) / sizeof(displays[0])); n++) {
505
-            if ((width == displays[n].width) && (height == displays[n].height)) {
506
-                unsigned char *data = [screen bitmapData];
507
-                [self visualizeSingleDisplay:n Data:data Width:width Height:height SPP:spp Alpha:alpha];
508
-                break;
509
-            }
510
-        }
668
+        [self visualizeThisImage:screen];
511 669
     }
512 670
     
513 671
     [self sendLEDFrame];
@@ -523,7 +681,8 @@ UInt8 ledColorData[LED_COUNT * 3];
523 681
     }
524 682
 #endif
525 683
     
526
-    timer = [NSTimer scheduledTimerWithTimeInterval:DISPLAY_DELAY target:self selector:@selector(visualizeDisplay:) userInfo:nil repeats:NO];
684
+    [self startAmbilightTimer];
527 685
 }
528
-
686
+*/
687
+ 
529 688
 @end

+ 1
- 1
DisplayBacklight/Info.plist Целия файл

@@ -21,7 +21,7 @@
21 21
 	<key>CFBundleSignature</key>
22 22
 	<string>????</string>
23 23
 	<key>CFBundleVersion</key>
24
-	<string>81</string>
24
+	<string>138</string>
25 25
 	<key>LSApplicationCategoryType</key>
26 26
 	<string>public.app-category.utilities</string>
27 27
 	<key>LSMinimumSystemVersion</key>

Loading…
Отказ
Запис