I found a workable answer, so I’m posting my findings for completeness. It remains to be seen if this solution will scale well with my program, but it seems promising.
I am going with an ‘alpha subtraction’ method. I made that term up, but I think you can do something like this in Photoshop with the right brush and settings. I grab the blue value of the mask and subtract that from the image’s current alpha value. This would produce the opposite effect of a regular alpha mask, so I also subtract the blue value from 255. Subtracting alpha from the image maintains transparency in the way that a regular alpha mask cannot, since it just overrides the value.
Below is demonstrative code. Comment out lines 32 or 33 or both to see the different options.
Here’s what it looks like with the alpha subtraction mask (line 32 commented out):
PGraphics pg, mask;
void setup() {
size(400, 400);
//draw a yellow background
background(#ffd700);
pg = createGraphics(width, height);
mask = createGraphics(width, height);
//draw 4 red circles over a transparent background
pg.beginDraw();
pg.clear();
pg.noStroke();
pg.fill(#D3191C);
pg.ellipse(width/2,height/2,50,50);
pg.ellipse(150,150,50,50);
pg.ellipse(100,100,50,50);
pg.ellipse(50,50,50,50);
pg.endDraw();
//create a square clipping mask
mask.beginDraw();
mask.background(0);
mask.noStroke();
mask.fill(255);
mask.rect(100,100,200,200);
mask.endDraw();
//mask the circles
//pg.mask(mask); // <- std function
alphaSubtract(pg, mask); // <- new function
//show the result
image(pg, 0, 0);
}
void draw() {
}
void alphaSubtract(PGraphics img, PGraphics cm){
img.loadPixels();
cm.loadPixels();
if(img.pixels.length != cm.pixels.length){
return;
}
for(int j = 0; j<img.height; j++){
for(int i = 0; i<img.width; i++){
// get argb values
color argb = img.pixels[(j*img.width) + i];
int a = argb >> 24 & 0xFF;
int r = argb >> 16 & 0xFF;
int g = argb >> 8 & 0xFF;
int b = argb & 0xFF;
color maskPixel = cm.pixels[(j*img.width) + i];
int alphaShift = 0xFF - (maskPixel & 0xFF); //grab blue value from mask pixel
// subtract alphaShift from pixel's alpha value;
img.pixels[(j*img.width) + i] = color(r,g,b,a-alphaShift);
}
}
}
void keyPressed(){
if(key == 's'){
save("with_mask.png");
}
}