C++, n=12 in less than 60 seconds on my laptop.

Try it online!

#include <iostream>
#include <vector>
#include <algorithm>
#include <chrono>
#include <bitset>
#include <string>

// Returns the Hamming distance (count of differing bits) between two bit patterns.
unsigned int hamming_distance(uint64_t a, uint64_t b) {
    return __builtin_popcountll(a ^ b);
}

// Returns the count of positions where both bit patterns have a '1'.
unsigned int common_ones(uint64_t a, uint64_t b) {
    return __builtin_popcountll(a & b);
}

// Checks if two rows (bit patterns) are "unfriendly":
// distance > number of common ones.
bool is_unfriendly(uint64_t a, uint64_t b) {
    return hamming_distance(a, b) > common_ones(a, b);
}

// Tries to build an unfriendly matrix by picking rows in descending order of their 1-count.
// - `rows_sorted` is a list of all bit patterns with their 1-count sorted descending.
// - `n` is the matrix dimension.
// - `chosen` holds currently picked row bit patterns.
void backtrack(
    const std::vector<std::pair<uint64_t, unsigned int>>& rows_sorted,
    size_t n,
    std::vector<uint64_t>& chosen,
    size_t start_index,
    std::vector<uint64_t>& best_solution,
    unsigned int& best_ones,
    std::chrono::seconds time_limit,
    std::chrono::steady_clock::time_point start_time
) {
    // If time exceeded, stop searching.
    if (std::chrono::steady_clock::now() - start_time > time_limit) {
        return;
    }

    // If we already have n rows, update the best solution if better.
    if (chosen.size() == n) {
        unsigned int sum_ones = 0;
        for (const auto& row : chosen) {
            sum_ones += __builtin_popcountll(row);
        }
        
        if (sum_ones > best_ones) {
            best_ones = sum_ones;
            best_solution = chosen;
        }
        return;
    }

    // Prune if the maximum possible 1s from here cannot beat current best:
    // (sum of chosen ones + sum of top row-ones in the future)
    unsigned int chosen_sum = 0;
    for (const auto& row : chosen) {
        chosen_sum += __builtin_popcountll(row);
    }
    
    unsigned int future_sum = 0;
    size_t remaining = n - chosen.size();
    for (size_t i = start_index; i < start_index + remaining && i < rows_sorted.size(); ++i) {
        future_sum += rows_sorted[i].second;
    }
    
    unsigned int max_possible = chosen_sum + future_sum;
    if (max_possible <= best_ones) {
        return;
    }

    // Try to pick rows from start_index onward.
    for (size_t i = start_index; i < rows_sorted.size(); ++i) {
        // Quick check for time limit again.
        if (std::chrono::steady_clock::now() - start_time > time_limit) {
            break;
        }

        const auto& [candidate_row, _] = rows_sorted[i];
        // Check if candidate_row is unfriendly with all chosen rows so far.
        bool valid = true;
        for (const auto& prev_row : chosen) {
            if (!is_unfriendly(candidate_row, prev_row)) {
                valid = false;
                break;
            }
        }
        // If it is unfriendly with all, pick it and recurse.
        if (valid) {
            chosen.push_back(candidate_row);
            backtrack(
                rows_sorted,
                n,
                chosen,
                i + 1,
                best_solution,
                best_ones,
                time_limit,
                start_time
            );
            chosen.pop_back();
        }
    }
}

int main(int argc, char* argv[]) {
    // Read n from command line or default to something for demonstration.
    if (argc < 2) {
        std::cerr << "Usage: " << argv[0] << " <n>" << std::endl;
        return 1;
    }
    
    size_t n = std::stoul(argv[1]);

    // For n up to 64, we can store each row in a 64-bit integer.
    // We will explore all possible 2^n patterns (feasible up to about n=12).
    if (n > 64) {
        std::cerr << "n = " << n << " is too large to be encoded in 64 bits." << std::endl;
        return 1;
    }

    // Generate all possible row bit patterns for n bits.
    // Store them alongside their count of 1s for sorting.
    std::vector<std::pair<uint64_t, unsigned int>> all_rows;
    uint64_t max_pattern = 1ULL << n;
    for (uint64_t row_pattern = 0; row_pattern < max_pattern; ++row_pattern) {
        all_rows.emplace_back(row_pattern, __builtin_popcountll(row_pattern));
    }

    // Sort patterns by descending number of 1s.
    std::sort(all_rows.begin(), all_rows.end(), 
        [](const auto& a, const auto& b) { return a.second > b.second; });

    // We'll store the best solution found (list of row patterns),
    // and the best sum of 1s so far.
    std::vector<uint64_t> best_solution;
    unsigned int best_ones = 0;

    // Set a time limit (e.g., 55 seconds, giving some buffer under 60).
    std::chrono::seconds time_limit(55);
    auto start_time = std::chrono::steady_clock::now();

    // Backtracking search.
    std::vector<uint64_t> chosen;
    backtrack(
        all_rows,
        n,
        chosen,
        0,
        best_solution,
        best_ones,
        time_limit,
        start_time
    );

    // Output the resulting matrix.
    // Each selected row bit pattern is output as '0/1' across n columns.
    std::vector<std::vector<int>> matrix(n, std::vector<int>(n, 0));
    for (size_t row_index = 0; row_index < best_solution.size(); ++row_index) {
        uint64_t pattern = best_solution[row_index];
        for (size_t col_index = 0; col_index < n; ++col_index) {
            // If the col_index-th bit is set, fill with 1.
            if (pattern & (1ULL << col_index)) {
                matrix[row_index][col_index] = 1;
            }
        }
    }

    // Print the matrix row by row.
    for (const auto& row : matrix) {
        for (size_t i = 0; i < row.size(); ++i) {
            std::cout << row[i];
            if (i < row.size() - 1) {
                std::cout << " ";
            }
        }
        std::cout << std::endl;
    }

    return 0;
}

Charcoal, 59 bytes, n=4 on ATO

Ｎθ≔ΦＥＸ²×θθ⪪﹪÷ιＸ²…⁰×θθ²θ⬤ι⬤…ιμ‹Σ×λνΣＥλ↔⁻π§νρη≔ＥηΣΣιζ⭆¹§η⌕ζ⌈ζ

Attempt This Online! Link is to verbose version of code. Explanation: Just brute force, so although ATO can solve n=4 in about half a minute, n=5 would take several hours.

Ｎθ

Input n.

≔ΦＥＸ²×θθ⪪﹪÷ιＸ²…⁰×θθ²θ⬤ι⬤…ιμ‹Σ×λνΣＥλ↔⁻π§νρη

Enumerate all binary square matrices of size n and filter on those that are unfriendly.

≔ＥηΣΣιζ⭆¹§η⌕ζ⌈ζ

Find and pretty-print the busiest of the unfriendly matrices.

86 bytes for a faster version that can do n=5 in minutes rather than hours:

Ｎθ≔ＥＸ²θ﹪÷ιＸ²⮌…⁰θ²ηＦη⊞υΦη‹Σ×ικΣ×⁺ιＸ⁰ι⁺Ｘ⁰κκ≔⟦⟦⟧⟧ζＦθ≔ΣＥηＥΦζ⬤μ№§υ↨ξ²κ⁺μ⟦κ⟧ζ≔ＥζΣΣιε⭆¹§ζ⌕ε⌈ε

Attempt This Online! Link is to verbose version of code. Explanation:

Ｎθ

Input n.

≔ＥＸ²θ﹪÷ιＸ²⮌…⁰θ²η

Create all of the bit patterns for a single row.

Ｆη⊞υΦη‹Σ×ικΣ×⁺ιＸ⁰ι⁺Ｘ⁰κκ

For each bit pattern, find those bit patterns that are unfriendly.

≔⟦⟦⟧⟧ζＦθ≔ΣＥηＥΦζ⬤μ№§υ↨ξ²κ⁺μ⟦κ⟧ζ

Starting from an empty matrix, build up rows by adding those bit patterns that are unfriendly to all the rows so far.

≔ＥζΣΣιε⭆¹§ζ⌕ε⌈ε

Find and pretty-print the busiest of the unfriendly matrices.

Python, n=12 in less than 60 seconds on my laptop.

import sys
import time

def hamming_distance(a, b):
    """
    Returns the Hamming distance (number of differing bits) between two integers a and b.
    """
    return bin(a ^ b).count('1')

def common_ones(a, b):
    """
    Returns the count of positions where a and b both have 1-bits.
    """
    return bin(a & b).count('1')

def is_unfriendly(a, b):
    """
    Checks if two rows a and b (bit patterns) are 'unfriendly':
    Hamming distance > number of positions where both have 1s.
    """
    return hamming_distance(a, b) > common_ones(a, b)

def backtrack(rows_sorted, n, chosen, start_idx, best_solution, best_ones, end_time):
    """
    Recursively attempts to build an unfriendly matrix by picking rows in descending order of 1-count.

    rows_sorted: list of (row_bit_pattern, count_ones)
    n: target number of rows in the solution
    chosen: currently picked row patterns
    start_idx: next index in rows_sorted to try
    best_solution: global best set of rows found
    best_ones: global best number of 1s found
    end_time: cutoff time to stop searching
    """
    # Check time limit
    if time.time() > end_time:
        return

    # If we have n rows, see if it improves our best solution
    if len(chosen) == n:
        current_sum = sum(bin(r).count('1') for r in chosen)
        if current_sum > best_ones[0]:
            best_ones[0] = current_sum
            best_solution[:] = chosen[:]
        return

    # Quick sum of chosen
    chosen_sum = sum(bin(r).count('1') for r in chosen)

    # Potential max if we pick the remaining top-ones rows
    # to see if we can beat the current best
    potential_max = chosen_sum + sum(v for _, v in rows_sorted[start_idx:start_idx + (n - len(chosen))])
    if potential_max <= best_ones[0]:
        # Prune if we can't do better
        return

    for i in range(start_idx, len(rows_sorted)):
        if time.time() > end_time:
            break
        row_val, _ = rows_sorted[i]

        # Check if it's unfriendly with all chosen rows
        valid = True
        for prev in chosen:
            if not is_unfriendly(row_val, prev):
                valid = False
                break

        # If valid, recurse
        if valid:
            chosen.append(row_val)
            backtrack(rows_sorted, n, chosen, i+1, best_solution, best_ones, end_time)
            chosen.pop()

def main():
    if len(sys.argv) < 2:
        print("Usage: python unfriendly.py <n>")
        sys.exit(1)

    n = int(sys.argv[1])
    
    # For n up to 64, we can store each row in a 64-bit integer.
    # The method only remains feasible for n roughly up to 12 due to 2^n growth.
    if n > 64:
        print(f"{n} is too large to represent in 64 bits.")
        sys.exit(1)

    # Generate all possible row bit patterns (from 0 to 2^n - 1)
    # and store them with their count of ones
    all_rows = []
    for row_pattern in range(1 << n):
        all_rows.append((row_pattern, bin(row_pattern).count('1')))

    # Sort rows by descending number of 1 bits
    all_rows.sort(key=lambda x: x[1], reverse=True)

    best_solution = []
    best_ones = [0]  # use list for mutability in function
    
    # You can adjust the search duration limit if desired
    start_time = time.time()
    time_limit_secs = 55
    end_time = start_time + time_limit_secs

    backtrack(all_rows, n, [], 0, best_solution, best_ones, end_time)

    # Convert the chosen row bit patterns into a 2D matrix
    matrix = []
    for row_val in best_solution:
        # build a list of bits '0' or '1'
        row_bits = [(row_val >> col) & 1 for col in range(n)]
        # If you want bits in left-to-right order as standard reading, reverse them:
        # row_bits.reverse()
        matrix.append(row_bits)

    # Print the matrix row by row
    for row in matrix:
        print(" ".join(map(str, row)))

if __name__ == "__main__":
    main()

Rust, n=12 in less than 60 seconds on my laptop.

use std::time::Instant;

/// Returns the Hamming distance (count of differing bits) between two bit patterns.
fn hamming_distance(a: u64, b: u64) -> u32 {
    (a ^ b).count_ones()
}

/// Returns the count of positions where both bit patterns have a '1'.
fn common_ones(a: u64, b: u64) -> u32 {
    (a & b).count_ones()
}

/// Checks if two rows (bit patterns) are "unfriendly":
/// distance > number of common ones.
fn is_unfriendly(a: u64, b: u64) -> bool {
    hamming_distance(a, b) > common_ones(a, b)
}

/// Tries to build an unfriendly matrix by picking rows in descending order of their 1-count.
/// - `rows_sorted` is a list of all bit patterns with their 1-count sorted descending.
/// - `n` is the matrix dimension.
/// - `chosen` holds currently picked row bit patterns.
fn backtrack(
    rows_sorted: &[(u64, u32)],
    n: usize,
    chosen: &mut Vec<u64>,
    start_index: usize,
    best_solution: &mut Vec<u64>,
    best_ones: &mut u32,
    time_limit: std::time::Duration,
    start_time: Instant,
) {
    // If time exceeded, stop searching.
    if Instant::now().duration_since(start_time) > time_limit {
        return;
    }

    // If we already have n rows, update the best solution if better.
    if chosen.len() == n {
        let sum_ones: u32 = chosen
            .iter()
            .map(|&r| r.count_ones())
            .sum();
        if sum_ones > *best_ones {
            *best_ones = sum_ones;
            *best_solution = chosen.clone();
        }
        return;
    }

    // Prune if the maximum possible 1s from here cannot beat current best:
    // (sum of chosen ones + sum of top row-ones in the future)
    let chosen_sum: u32 = chosen.iter().map(|&r| r.count_ones()).sum();
    let max_possible = chosen_sum
        + rows_sorted[start_index..]
            .iter()
            .take(n - chosen.len())
            .map(|&(_, ones)| ones)
            .sum::<u32>();
    if max_possible <= *best_ones {
        return;
    }

    // Try to pick rows from start_index onward.
    for i in start_index..rows_sorted.len() {
        // Quick check for time limit again.
        if Instant::now().duration_since(start_time) > time_limit {
            break;
        }

        let (candidate_row, _) = rows_sorted[i];
        // Check if candidate_row is unfriendly with all chosen rows so far.
        let mut valid = true;
        for &prev_row in &*chosen {
            if !is_unfriendly(candidate_row, prev_row) {
                valid = false;
                break;
            }
        }
        // If it is unfriendly with all, pick it and recurse.
        if valid {
            chosen.push(candidate_row);
            backtrack(
                rows_sorted,
                n,
                chosen,
                i + 1,
                best_solution,
                best_ones,
                time_limit,
                start_time,
            );
            chosen.pop();
        }
    }
}

fn main() {
    // Read n from command line or default to something for demonstration.
    let args: Vec<String> = std::env::args().collect();
    if args.len() < 2 {
        eprintln!("Usage: {} <n>", args[0]);
        return;
    }
    let n: usize = args[1].parse().expect("Invalid integer for n");

    // For n up to 64, we can store each row in a 64-bit integer.
    // We will explore all possible 2^n patterns (feasible up to about n=12).
    if n > 64 {
        eprintln!("n = {} is too large to be encoded in 64 bits.", n);
        return;
    }

    // Generate all possible row bit patterns for n bits.
    // Store them alongside their count of 1s for sorting.
    let mut all_rows = Vec::new();
    for row_pattern in 0..(1u64 << n) {
        all_rows.push((row_pattern, row_pattern.count_ones()));
    }

    // Sort patterns by descending number of 1s.
    all_rows.sort_by(|a, b| b.1.cmp(&a.1));

    // We'll store the best solution found (list of row patterns),
    // and the best sum of 1s so far.
    let mut best_solution = Vec::new();
    let mut best_ones = 0;

    // Set a time limit (e.g., 55 seconds, giving some buffer under 60).
    let time_limit = std::time::Duration::from_secs(55);
    let start_time = Instant::now();

    // Backtracking search.
    let mut chosen = Vec::new();
    backtrack(
        &all_rows,
        n,
        &mut chosen,
        0,
        &mut best_solution,
        &mut best_ones,
        time_limit,
        start_time,
    );

    // Output the resulting matrix.
    // Each selected row bit pattern is output as '0/1' across n columns.
    let mut matrix = vec![vec![0; n]; n];
    for (row_index, &pattern) in best_solution.iter().enumerate() {
        for col_index in 0..n {
            // If the col_index-th bit is set, fill with 1.
            // Note that bit 0 is the least significant bit, so we check that carefully.
            if (pattern & (1 << col_index)) != 0 {
                matrix[row_index][col_index] = 1;
            }
        }
    }

    // Print the matrix row by row.
    // If you want columns in "left to right" order matching bits from left to right:
    // you can invert the indexing in the loop above.
    for row in matrix {
        println!(
            "{}",
            row.iter()
                .map(|&bit| bit.to_string())
                .collect::<Vec<String>>()
                .join(" ")
        );
    }
}