Python 3, 157 bytes

a,x,y=input().split()
i=int
b=bin
print(i(''.join(([c for c,d in zip(b(i(a)),b(i(y)))if d=='1'],[c+d for c,d in zip(b(i(a))[2:],b(i(y))[2:])])['¢'==x]),2))

C, 74 bytes

x;f(a,o,b){return+a|b?x=f(a/2,o,b/2),o-162?b&1?x*2|a&1:x:4*x|2*a&2|b&1:0;}

This is a recursive function whose arguments are first operand, operation (as a Unicode character) and second operand, returning the result.

How it works:

int f(const int a, const int op, const int b)
{
    if (a || b) {
        /* recurse first for higher bits */
        const int x = f(a >> 1, op, b >> 1);

        if (op == L'¢') {
            /* mingle */
            return (x << 2) + ((a&1) << 1) + (b&1);
        }

        /* else select */
        if (b&1) {
            /* selected bit */
            return (x << 1) + (a&1);
        } else {
            /* unselected bit */
            return x;
        }
    }
    /* both (0 ¢ 0) and (0 ~ 0) yield 0 */
    return 0;
}

Demo/test, using examples from the question:

#include <locale.h>
#include <stdio.h>

static int t(int a, int op, int b, int expected)
{
    int actual = f(a, op, b);
    if (actual == expected) { return 0; }
    fprintf(stderr, "FAIL! %u %lc %u = %u != %u\n",
            a, op, b, actual, expected);
    return 1;
}

int main(void)
{
    setlocale(LC_ALL, "");
    return t(234, L'¢', 4321, 16841865)
        +  t(2345, L'~', 7245, 37)
        +  t(5, L'¢', 6, 54)
        +  t(5, L'~', 6, 2)
        +  t(51234, L'~', 60003, 422)
        +  t(51234, L'¢', 60003, 4106492941);
}

Note: on GCC with Linux ABI and no optimisation, we get to 69 bytes by assigning to a instead of returning:

x;f(a,o,b){a=a|b?x=f(a/2,o,b/2),o-162?b&1?x*2|a&1:x:4*x|2*a&2|b&1:0;}

Pip, 32 bytes

FB(bQ'~?J$X*Z_;WV_;S*TB[ac]+E16)

Try It Online!

Takes input on the command line. The interpreter treats the space-separated inputs as separate command-line arguments. If that's too sneaky, here's a 37-byte version that takes a single string and splits on spaces manually.

Explanation

FB(bQ'~?J$X*Z_;WV_;S*TB[ac]+E16)
                       [ac]       List of the first and third arguments
                           +E16   Add 2^16 to each
                     TB           Convert each to binary
                   S*             Take off the first character of each
                                  This converts both numbers to padded 16-bit binary
   bQ'~                           Is the second argument "~"?
       ?                          If so, this function performs the select operation:
            Z_                      Zip the two binary numbers together
         $X*                        Fold each pair of bits on string repetition
                                    That is, if the second number's bit is 1, produce
                                    the first number's bit; otherwise, produce ""
        J                           Join that list of results into a single string
              ;                   If not, this function performs the mingle operation:
               WV_                  Interleave the two binary numbers
  (               ;            )  Call the selected function on the binary inputs
FB                                Convert the result from binary to decimal

Python 3, 174 166 148 126

Pretty straightforward, string operations, then conversion back to integer.

Limited to numbers which in binary have 99 digits (max 2⁹⁹-1 = 633825300114114700748351602687).

Thanks, Sp3000 and Vioz!

a,o,b=input().split()
print(int(''.join([(i+j,i[:j>'0'])[o>'~']for i,j in zip(*[bin(int(j))[2:].zfill(99)for j in(a,b)])]),2))

Or 165 chars, without limit:

a,o,b=input().split()
a,b=[bin(int(j))[2:]for j in(a,b)]
print(int(''.join([(i if j=='1'else'')if o=='~'else i+j for i,j in zip(a.zfill(len(b)),b.zfill(len(a)))]),2))

Ungolfed:

a, op, b = input().split()
a, b = [bin(int(j))[2:] for j in(a,b)] #convert to int (base 10), then to binary, remove leading '0b'
m = max(len(a), len(b))
a = a.zfill(m) #fill with leading zeroes
b = b.zfill(m)
if op == '~':
    ret = [i if j=='1' else'' for i, j in zip(a, b)]
else:
    ret = [i + j for i, j in zip(a, b)]
ret = ''.join(ret) #convert to string
ret = int(ret, 2) #convert to integer from base 2
print(ret)

Go, 186 bytes

func c(a,b uint)(o uint){for i:=0;i<16;i++{o|=(a&1<<(2*i+1))|(b&1<<(2*i))
a>>=1
b>>=1}
return}
func s(a,b uint)(o uint){for i,j:=0,0;i<16;i++{if b&1>0{o|=a&1<<j;j++}
a>>=1
b>>=1}
return}

Attempt This Online!

c is interleave, and s is select.

Explanations

Interleave

func c(a,b uint)(o uint){       // init return value in `o` with zero value
for i:=0;i<16;i++{              // for each bit (in the uint16)...
o|=                             // OR to `o`...
   (a&1                         // the last bit of `a`
       <<                       // shifted left by
         (2*i+1))               // the current bit * 2 + 1
                 |              // OR-ed with
                  (b&1          // the last bit of `b`
                      <<        // shifted left by
                        (2*i))  // the current bit * 2
a>>=1;b>>=1}                    // discard the last bit of `a` and `b`
return}

Select

func s(a,b uint)(o uint){ // init return value in `o` with zero value
for i,j:=0,0;i<16;i++{    // for each bit (in the uint16)...
if b&1==1{                // if the current bit in `b` is 1...
          o|=             // OR to `o`...
             a&1          // the last bit of `a`
                <<j       // shifted over by the number of 1s so far
                   ;j++}  // increment the number of 1s
a>>=1;b>>=1}              // discard last bit of `a` and `b`
return}

Vyxal, 201 bits^v2, 25.125 bytes

⌈sṫ£⌊vΠ:ḣ∆Z$tJvf⌊÷¥\¢=[Y|Tİ]B

Try it Online!

Bitstring:

110010100100100000011011100010011001001100101100100000110010011100111001000101011101011101101110001000110011011101000000100101011000110100110001000010010001101001011101001011100110111111100100110010111

Possibly golfable?

JavaScript (ES6), 103 117 119 124

Edit now working with numbers instead of strings

(not counting leading spaces, newlines and comments)

Test running the snippet on any EcmaScript 6 compliant browser (notably not Chrome not MSIE. I tested on Firefox, Safari 9 could go)

I=s=>
  (i=>{
    for(m=r=0,[a,o,b]=s.split` `;i>0;i<<=1) // loop until bit 31 of i is set
      o>'~'?r+=(b&i)*i+(a&i)*2*i:b&i?r+=(a&i)>>m:++m
  })(1)||r


// TEST
out=x=>O.innerHTML+=x+'\n\n';

[ ['234 ¢ 4321', 16841865], ['2345 ~ 7245', 37]
, ['5 ¢ 6', 54], ['5 ~ 6', 2]
, ['51234 ¢ 60003',4106492941], ['51234 ~ 60003', 422]]
.forEach(([i,o,r=I(i)])=>{
  out('Test '+ (o==r?'OK':'Fail')+'\nInput:    '+ i+'\nResult:   '+r+'\nExpected: '+o)})

<pre id=O></pre>

Python 2, 115 112 bytes

x,y,z=input().split()
d=y<""
f=lambda a,b:a+b and(b%2+5&4-d)*f(a/2,b/2)+(a%2*2+b%2)/3**d
print f(int(x),int(z))

The string on the second line contains a single unprintable character \x7d, the next char after ~.

All hopes of a nice, single lambda get crushed by the input format. There's probably a better way to read in input. Input like "51234 ¢ 60003" via STDIN.

The function f combines the following two recursive functions:

g=lambda a,b:a+b and 4*g(a/2,b/2)+a%2*2+b%2    # ¢
h=lambda a,b:a+b and(b%2+1)*h(a/2,b/2)+a*b%2   # ~

(-3 bytes with the help of @xnor)

Mathematica, 155 bytes

f=IntegerDigits[#,2,16]&;
g=#~FromDigits~2&;
¢=g[f@#~Riffle~f@#2]&;
s=g@Cases[Thread@{f@#,f@#2},{x_,1}->x]&;
ToExpression@StringReplace[#,{" "->"~","~"->"s"}]&

Evaluates to an anonymous function taking the string as input. Line breaks added for clarity.

f and g convert to/from base 2. Riffle does exactly what interleave is supposed to. I wanted to use Select for select but Cases is better unfortunately. The last line is a bit of trickery; spaces are changed to ~ which is Mathematica's infix operator, then the string is eval'd.

CJam, 31 bytes

rrc\r]{i2bF0e[}%(7=\zf{_)*?~}2b

Try it online in the CJam interpreter.

How it works

rr                              e# Read two tokens from STDIN.
  c\                            e# Cast the second to char and swap with the first.
    r                           e# Read a third token from STDIN.
     ]                          e# Wrap everything in an array.
      {       }%                e# For all three elements:
       i2b                      e#   Cast to int and convert to base 2.
          F0e[                  e#   Left-pad with zeroes to complete 15 digits.
                (               e# Shift out the first base 2 array.
                 7=             e# Select its eighth MSB (1 for '¢', 0 for '~').
                   \            e# Swap with the array of base 2 arrays.
                    z           e# Zip to transpose rows with columns.
                     f{     }   e# For each pair of base 2 digits:
                                e#   Push the bit, then the pair.
                       _        e#   Copy the pair.
                        )       e#   Pop the second digit.
                         *      e#   Repeat the first digit that many times.
                          ?     e#   Ternary if. Select the pair if the bit is
                                e#    truthy, the repeated first bit if it's falsy.
                           ~    e#   Dump the selected array on the stack.
                             2b e# Convert from base 2 to integer.

Javascript ES6 (3 arguments) 141 138 136 121 119 bytes

b=x=>(65536|x).toString`2`
f=(x,o,y)=>+eval(`'0b'+(b(y)+b(x)).replace(/^1|${o=='~'?1:'(.)'}(?=.{16}(.)())|./g,'$2$1')`)

Test:

;[f(234,'¢',4321),f(2345,'~',7245)]=="16841865,37"

Javascript ES6 (1 argument) 135 133 bytes

b=x=>(65536|x).toString`2`
f=s=>([x,o,y]=s.split` `)|eval(`'0b'+(b(y)+b(x)).replace(/^1|${o=='~'?1:'(.)'}(?=.{16}(.)())|./g,'$2$1')`)

Test:

;[f('234 ¢ 4321'),f('2345 ~ 7245')]=="16841865,37"

PS: New line is counted as 1 byte as it can be replaced by ;.

Haskell, 77

g=(`mod`2)
h=(`div`2)
0¢0=0
a¢b=g a+2*b¢h a
a?0=0
a?b=g a*g b+(1+g b)*h a?h b

input is given by applying the input to the functions/operators ? and ¢ defined in the code (Haskell can't define an operator ~ for technical reasons).

basically works the old recursive approach.

R, 145 bytes

s=scan(,"");a=as.double(c(s[1],s[3]));i=intToBits;cat(packBits(if(s[2]=="~")c(i(a[1])[i(a[2])>0],i(0))[1:32] else c(rbind(i(a[2]),i(a[1]))),"i"))

Ungolfed + explanation:

# Read a string from STDIN and split it on spaces
s <- scan(, "")

# Convert the operands to numeric
a <- as.double(c(s[1], s[3]))

o <- if (s[2] == "~") {
    # Get the bits of the first operand corresponding to ones in
    # the second, right pad with zeros, and truncate to 32 bits
    c(intToBits(a[1])[intToBits(a[2]) == 1], intToBits(0))[1:32]
} else {
    # Interleave the arrays of bits of the operands
    c(rbind(intToBits(a[2]), intToBits(a[1])))
}

# Make an integer from the raw bits and print  it to STDOUT
cat(packBits(o, "integer"))

Pyth, 32 31 29 bytes

isummFdG}\~zCm.[Z16jvd2%2cz)2

Try it online: Regular Input / Test Suite

Thanks to @isaacg for golfing off one byte.

Explanation:

                         cz)   split input at spaces
                       %2      only take every second item (the numbers)
             m                 map each number d to:
                    vd           convert d to int
                   j  2          convert to base 2
              .[Z16              pad zeros at the left
            C                  zip
  u     }\~z                   apply the following function ("~" in input) times:
   m   G                         map each pair d to:
    mFd                          convert [x,0] to [] and [x,1] to [x]
 s                             take sum (unfold all lists)
i                           2  convert back from base 2 and print

CJam, 61 50 46 41 34 bytes

Thanks @Dennis for pointing out a 4 bytes golf.

rrc'~=:X;r]{i2bF0e[}/.{X{{;}|}&}2b

Try it online.

K5, 53 52 bytes

{b/({,/x,'y};{x@&y})[*"~"=y][b\.x;(b:20#2)\.z]}." "\

53-byte version:

{b/({,/x,'y};{x@&y})[*"¢~"?y][b\.x;(b:20#2)\.z]}." "\

Still needs a bit more golfing.

Matlab, 119 113 bytes

function f(s)
t=dec2bin(str2double(strsplit(s,{'¢' '~'}))');u=any(s>'~');[~u u]*bin2dec({t(1,t(2,:)==49) t(:)'})

Ungolfed:

function f(s)                                     % input s is a string
t = dec2bin(str2double(strsplit(s,{'¢' '~'}))');  % get the two numbers and convert to
                                                  % two-row char array of zeros of ones
u = any(s>'~');                                   % 1 indicates '¢'; 0 indicates '~'
[~u u]*bin2dec({t(1,t(2,:)==49) t(:)'})           % compute both results and display
                                                  % that indicated by u

Examples:

>> f('234 ¢ 4321')
ans =
    16841865

>> f('2345 ~ 7245')
ans =
    37

C, 127 123 bytes + 5 penalty = 128

scanf counts the unicode symbol as more than one character which complicates things a lot, so I'm applying the 5-byte penalty for using $.

a,b,q,x,i;main(){scanf("%d %c %d",&a,&q,&b);for(i=65536;i/=2;)q%7?x=x*4|a/i*2&2|b/i&1:b/i&1&&(x=x*2|a/i&1);printf("%u",x);}

The changes from the original version are:

-The test for $ or ~ has been revised from q&2 to q%7. This reverses the true/false values, allowing the code for $ operator to go before the : which means a set of parentheses can be eliminated.

-The i loop now counts down in powers of 2 which is longer, but permits >> to be substituted by / and saves some parentheses.

Original version 127 bytes

a,b,q,x,i;
main(){
  scanf("%d %c %d",&a,&q,&b);
  for(i=16;i--;)
    q&2?
      b>>i&1&&(x=x*2|a>>i&1):    // ~ operator. && used as conditional: code after it is executed only if code before returns truthy.
      (x=x*4|(a>>i&1)*2|b>>i&1); // $ operator
  printf("%u",x);
}

I went with a single loop with the conditionals inside to avoid the overhead of two loops. In both cases I rightshift the bits of the operands down to the 1's bit, and build up the result from the most significant to least significant bit, leftshifting the result (multiplying by 2 or 4) as I go.

Pyth, 43 bytes

Part of me feels nervous posting such a long Pyth answer on isaacg's question... :oP

J.(Kczd1Am.BvdKiu?qJ\~u+G?qeH\1hHk+VGHk.iGH2

Explaination:

                                               Implicit: z=input(), k='', d=' '
   Kczd                                        Split z on spaces, store in K
J.(    1                                       Remove centre element from K, store in J
         m    K                                For each d in K
          .Bvd                                 Evaluate as int, convert to binary string
        A                                      Store pair in G and H
                                               ~ processing:
                                 +VGH          Create vectorised pairs ([101, 110] -> [11, 01, 10])
                     u               k         Reduce this series, starting with empty string
                        ?qeH\1                 If 2nd digit == 1...
                              hHk              ... take the 1st digit, otherwise take ''
                      +G                       Concatenate
                                      .iGH     ¢ processing: interleave G with H
                ?qJ\~                          If J == ~, take ~ processing, otherwise take ¢
               i                          2    Convert from binary to decimal

J, 173

f=:|."1@(>@(|.&.>)@(#:@{:;#:@{.))
m=:2&#.@:,@:|:@:|.@:f
s=:2&#.@#/@:f
a=:{&a.@-.
(1!:2)&2(s@".@:a&126)^:(126 e.i)((m@".@:a&194 162)^:(1 e.194 162 E.i)i=._1}.(a.i.((1!:1)3)))

expects one line of input

input expected to terminate after new line with EOF