Image Light Feature - GSOC 2023

src/webgl/light.js

@@ -494,6 +494,12 @@ p5.prototype.pointLight = function(v1, v2, v3, x, y, z) {
   return this;
 };
 
+p5.prototype.imageLight = function(img){
+  // setting activeImageLight so that the setUniform is executed
+  this._renderer.activeImageLight = img;
+  this._renderer._enableLighting = true;
+};
+
 /**
  * Places an ambient and directional light in the scene.
  * The lights are set to ambientLight(128, 128, 128) and

src/webgl/p5.RendererGL.js

@@ -9,6 +9,7 @@ import './p5.Matrix';
 import './p5.Framebuffer';
 import { readFileSync } from 'fs';
 import { join } from 'path';
+import { MipmapTexture } from './p5.Texture';
 
 const STROKE_CAP_ENUM = {};
 const STROKE_JOIN_ENUM = {};
@@ -67,17 +68,21 @@ const defaultShaders = {
   phongFrag:
     lightingShader +
     readFileSync(join(__dirname, '/shaders/phong.frag'), 'utf-8'),
-  fontVert: webgl2CompatibilityShader +
-    readFileSync(join(__dirname, '/shaders/font.vert'), 'utf-8'),
-  fontFrag: webgl2CompatibilityShader +
-    readFileSync(join(__dirname, '/shaders/font.frag'), 'utf-8'),
+  fontVert: readFileSync(join(__dirname, '/shaders/font.vert'), 'utf-8'),
+  fontFrag: readFileSync(join(__dirname, '/shaders/font.frag'), 'utf-8'),
   lineVert:
     lineDefs + readFileSync(join(__dirname, '/shaders/line.vert'), 'utf-8'),
   lineFrag:
     lineDefs + readFileSync(join(__dirname, '/shaders/line.frag'), 'utf-8'),
   pointVert: readFileSync(join(__dirname, '/shaders/point.vert'), 'utf-8'),
-  pointFrag: readFileSync(join(__dirname, '/shaders/point.frag'), 'utf-8')
+  pointFrag: readFileSync(join(__dirname, '/shaders/point.frag'), 'utf-8'),
+  imageLightVert : readFileSync(join(__dirname, '/shaders/imageLight.vert'), 'utf-8'),
+  imageLightDiffusedFrag : readFileSync(join(__dirname, '/shaders/imageLightDiffused.frag'), 'utf-8'),
+  imageLightSpecularFrag : readFileSync(join(__dirname, '/shaders/imageLightSpecular.frag'), 'utf-8')
 };
+for (const key in defaultShaders) {
+  defaultShaders[key] = webgl2CompatibilityShader + defaultShaders[key];
+}
 
 const filterShaderFrags = {
   [constants.GRAY]:
@@ -468,6 +473,17 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     this.spotLightAngle = [];
     this.spotLightConc = [];
 
+    // Null if no image light is set, otherwise, it is set to a p5.Image
+    // made a blurrytexture attribute to be used in imageLight function
+    // this is to lookup image from blurrytexture Map rather from a
+    // texture map
+    this.diffusedTextures = new Map();
+    // map to store the created textures to prevent mis calculation
+    this.specularTextures = new Map();
+    // property to be set true after imageLight is called
+    // then if it is true the setUniform would be done on shader
+    this.activeImageLight = null;
+
     this.drawMode = constants.FILL;
 
     this.curFillColor = this._cachedFillStyle = [1, 1, 1, 1];
@@ -601,6 +617,7 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     this.curStrokeCap = constants.ROUND;
     this.curStrokeJoin = constants.ROUND;
 
+    // repetition
     // map of texture sources to textures created in this gl context via this.getTexture(src)
     this.textures = new Map();
 
@@ -1740,14 +1757,18 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
       if (this._pInst._glAttributes.perPixelLighting) {
         this._defaultLightShader = new p5.Shader(
           this,
-          defaultShaders.phongVert,
-          defaultShaders.phongFrag
+          this._webGL2CompatibilityPrefix('vert', 'highp') +
+            defaultShaders.phongVert,
+          this._webGL2CompatibilityPrefix('frag', 'highp') +
+            defaultShaders.phongFrag
         );
       } else {
         this._defaultLightShader = new p5.Shader(
           this,
-          defaultShaders.lightVert,
-          defaultShaders.lightTextureFrag
+          this._webGL2CompatibilityPrefix('vert', 'highp') +
+            defaultShaders.lightVert,
+          this._webGL2CompatibilityPrefix('frag', 'highp') +
+            defaultShaders.lightTextureFrag
         );
       }
     }
@@ -1759,8 +1780,10 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     if (!this._defaultImmediateModeShader) {
       this._defaultImmediateModeShader = new p5.Shader(
         this,
-        defaultShaders.immediateVert,
-        defaultShaders.vertexColorFrag
+        this._webGL2CompatibilityPrefix('vert', 'mediump') +
+          defaultShaders.immediateVert,
+        this._webGL2CompatibilityPrefix('frag', 'mediump') +
+          defaultShaders.vertexColorFrag
       );
     }
 
@@ -1771,8 +1794,10 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     if (!this._defaultNormalShader) {
       this._defaultNormalShader = new p5.Shader(
         this,
-        defaultShaders.normalVert,
-        defaultShaders.normalFrag
+        this._webGL2CompatibilityPrefix('vert', 'mediump') +
+          defaultShaders.normalVert,
+        this._webGL2CompatibilityPrefix('frag', 'mediump') +
+          defaultShaders.normalFrag
       );
     }
 
@@ -1783,8 +1808,10 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     if (!this._defaultColorShader) {
       this._defaultColorShader = new p5.Shader(
         this,
-        defaultShaders.normalVert,
-        defaultShaders.basicFrag
+        this._webGL2CompatibilityPrefix('vert', 'mediump') +
+          defaultShaders.normalVert,
+        this._webGL2CompatibilityPrefix('frag', 'mediump') +
+          defaultShaders.basicFrag
       );
     }
 
@@ -1795,8 +1822,10 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     if (!this._defaultPointShader) {
       this._defaultPointShader = new p5.Shader(
         this,
-        defaultShaders.pointVert,
-        defaultShaders.pointFrag
+        this._webGL2CompatibilityPrefix('vert', 'mediump') +
+          defaultShaders.pointVert,
+        this._webGL2CompatibilityPrefix('frag', 'mediump') +
+          defaultShaders.pointFrag
       );
     }
     return this._defaultPointShader;
@@ -1806,8 +1835,10 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     if (!this._defaultLineShader) {
       this._defaultLineShader = new p5.Shader(
         this,
-        defaultShaders.lineVert,
-        defaultShaders.lineFrag
+        this._webGL2CompatibilityPrefix('vert', 'mediump') +
+          defaultShaders.lineVert,
+        this._webGL2CompatibilityPrefix('frag', 'mediump') +
+          defaultShaders.lineFrag
       );
     }
 
@@ -1875,6 +1906,81 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     return tex;
   }
 
+  // used in imageLight
+  // create a new texture from the img input
+  /*
+  To create a blurry image from the input non blurry img
+  Add it to the blurryTexture map
+  Returns the blurry image
+  */
+  getBlurryTexture(input){
+    // if one already exists for a given input image
+    if(this.diffusedTextures.get(input)!=null){
+      return this.diffusedTextures.get(input);
+    }
+    // if not, then create one
+    let newGraphic; // maybe switch to framebuffer
+    // draw the blurry image
+    // set the shader on the graphic and set the shader on the image
+    // this._renderer._applyTextProperties(img, newGraphic);
+    // make small width, hardcode to 200px
+    let smallWidth = 200;
+    //let smallWidth = input.width;
+    let width = smallWidth;
+    let height = Math.floor(smallWidth * (input.height / input.width));
+    newGraphic = createGraphics(width, height, WEBGL);
+    // create graphics is like making a new sketch, all functions on main
+    // sketch it would be available on graphics
+    let irradiance = newGraphic.createShader(
+      defaultShaders.imageLightVert,
+      defaultShaders.imageLightDiffusedFrag
+    );
+    newGraphic.shader(irradiance);
+    irradiance.setUniform('environmentMap', input);
+    newGraphic.noStroke();
+    newGraphic.rectMode(newGraphic.CENTER);
+    newGraphic.rect(0, 0, newGraphic.width, newGraphic.height);
+    this.diffusedTextures.set(input, newGraphic);
+    return newGraphic;
+  }
+
+  getSpecularTexture(input){
+    // check if already exits (there are tex of diff resolution so which one to check)
+    // currently doing the whole array
+    // please check this line ?
+    if(this.specularTextures.get(input)!=null){
+      return this.specularTextures.get(input);
+    }
+    // What should be the size?
+    const size = 512;
+    let tex;
+    const levels = [];
+    const graphic = createGraphics(size, size, WEBGL);
+    let count = Math.log(size)/Math.log(2);
+    graphic.pixelDensity(1);
+    for (let w = size; w >= 1; w /= 2) {
+      graphic.resizeCanvas(w, w);
+      let currCount = Math.log(w)/Math.log(2);
+      let roughness = 1-currCount/count;
+      let myShader = graphic.createShader(
+        defaultShaders.imageLightVert,
+        defaultShaders.imageLightSpecularFrag
+      );
+      graphic.shader(myShader);
+      graphic.clear();
+      myShader.setUniform('environmentMap', input);
+      myShader.setUniform('roughness', roughness);
+      graphic.noStroke();
+      graphic.plane(w, w);
+      levels.push(graphic.get().drawingContext.getImageData(0, 0, w, w));
+      // please check this line?
+    }
+    graphic.remove();
+    tex = new MipmapTexture(this, levels, {});
+    this.specularTextures.set(input,tex);
+    return tex;
+  }
+
   /**
    * @method activeFramebuffer
    * @private
@@ -1920,6 +2026,9 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     fillShader.setUniform('uEmissive', this._useEmissiveMaterial);
     fillShader.setUniform('uShininess', this._useShininess);
 
+    // calling the _setImageLightUniforms from here
+    this._setImageLightUniforms(fillShader);
+
     fillShader.setUniform('uUseLighting', this._enableLighting);
 
     const pointLightCount = this.pointLightDiffuseColors.length / 3;
@@ -1970,6 +2079,31 @@ p5.RendererGL = class RendererGL extends p5.Renderer {
     fillShader.bindTextures();
   }
 
+  // getting called from _setFillUniforms
+  _setImageLightUniforms(shader){
+    //set uniform values
+    if( this.activeImageLight == null ){
+      console.log('activeImageLight prop is null');
+    }
+    shader.setUniform('uUseImageLight', this.activeImageLight != null );
+    // true
+    if (this.activeImageLight) {
+      // this.activeImageLight has image as a key
+      // look up the texture from the blurryTexture map
+      // change the name of shader of diffused to prevent confusion
+      let diffusedLight = this.getBlurryTexture(this.activeImageLight);
+      // change the name of this uniform as diff for diffused and specular
+      shader.setUniform('environmentMapDiffused', diffusedLight);
+      let specularLight = this.getSpecularTexture(this.activeImageLight);
+      // shine >= 1 so
+      // 0-1*8 = 0-8
+      // 0-8 * 20 = 0-160
+      let roughness = 20/this._useShininess;
+      shader.setUniform('levelOfDetail', roughness*8);
+      shader.setUniform('environmentMapSpecular', specularLight);
+    }
+  }
+
   _setPointUniforms(pointShader) {
     pointShader.bindShader();
 

src/webgl/p5.Texture.js

@@ -337,24 +337,24 @@ p5.Texture = class Texture {
   setInterpolation (downScale, upScale) {
     const gl = this._renderer.GL;
 
-    if (downScale === constants.NEAREST) {
-      this.glMinFilter = gl.NEAREST;
-    } else {
-      this.glMinFilter = gl.LINEAR;
-    }
-
-    if (upScale === constants.NEAREST) {
-      this.glMagFilter = gl.NEAREST;
-    } else {
-      this.glMagFilter = gl.LINEAR;
-    }
+    this.glMinFilter = this.glFilter(downScale);
+    this.glMagFilter = this.glFilter(upScale);
 
     this.bindTexture();
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, this.glMinFilter);
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, this.glMagFilter);
     this.unbindTexture();
   }
 
+  glFilter(filter) {
+    const gl = this._renderer.GL;
+    if (filter === constants.NEAREST) {
+      return gl.NEAREST;
+    } else {
+      return gl.LINEAR;
+    }
+  }
+
   /**
    * Sets the texture wrapping mode. This controls how textures behave
    * when their uv's go outside of the 0 - 1 range. There are three options:
@@ -452,6 +452,51 @@ p5.Texture = class Texture {
   }
 };
 
+export class MipmapTexture extends p5.Texture {
+  constructor(renderer, levels, settings) {
+    super(renderer, levels, settings);
+    const gl = this._renderer.GL;
+    if (this.glMinFilter === gl.LINEAR) {
+      this.glMinFilter = gl.LINEAR_MIPMAP_LINEAR;
+    }
+  }
+
+  glFilter(_filter) {
+    const gl = this._renderer.GL;
+    // TODO: support others
+    return gl.LINEAR_MIPMAP_LINEAR;
+  }
+
+  _getTextureDataFromSource() {
+    return this.src;
+  }
+
+  init(levels) {
+    const gl = this._renderer.GL;
+    this.glTex = gl.createTexture();
+
+    this.bindTexture();
+    for (let level = 0; level < levels.length; level++) {
+      gl.texImage2D(
+        this.glTarget,
+        level,
+        this.glFormat,
+        this.glFormat,
+        this.glDataType,
+        levels[level]
+      );
+    }
+
+    this.glMinFilter = gl.LINEAR_MIPMAP_LINEAR;
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, this.glMagFilter);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, this.glMinFilter);
+
+    this.unbindTexture();
+  }
+
+  update() {}
+}
+
 export function checkWebGLCapabilities({ GL, webglVersion }) {
   const gl = GL;
   const supportsFloat = webglVersion === constants.WEBGL2